:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
import dataclasses
from collections.abc import Iterable, Iterator
from datetime import UTC, datetime, timedelta
from typing import Any, cast
from requests.exceptions import HTTPError
from products.warehouse_sources.sources.sdk import (
ClientConfig,
Endpoint,
EndpointResource,
JSONResponseCursorPaginator,
RESTAPIConfig,
RESTClient,
ResumableSourceManager,
SourceInputs,
SourceResponse,
build_dependent_resource,
create_auth,
frozen,
parse_datetime_value,
rest_api_resource,
schema_for_resource,
)
from products.warehouse_sources.sources.sim._config import SimSourceConfig
from products.warehouse_sources.sources.sim.settings import ENDPOINTS, PARENT_ENDPOINTS, USAGE_WINDOW_DAYS, SimEndpoint
# Page size for listing organizations while validating usage access.
USAGE_PROBE_ORGANIZATIONS = 10
AUTH_ERROR = "Your Sim API key is invalid or expired. Create a new key in Sim settings and reconnect."
PERMISSION_ERROR = "Your Sim API key cannot read this workspace. Check the key's workspace access and permissions."
WORKSPACE_ERROR = "Sim could not find this workspace. Check the workspace ID and your API key's access."
USAGE_PERMISSION_ERROR = (
"Sim usage events need a personal API key from an organization admin. Workspace API keys cannot read them."
)
ALL_ENDPOINTS: dict[str, SimEndpoint] = {**ENDPOINTS, **PARENT_ENDPOINTS}
PAGE_EXTRA: Endpoint = {
"data_selector": "data",
"data_selector_required": True,
"paginator": {"type": "cursor", "cursor_path": "nextCursor", "cursor_param": "cursor"},
}
@frozen
class SimResumeConfig:
cursor: str | None = None
fanout_state: dict[str, Any] | None = None
# A usage cursor is only valid for the window it was issued for, so a resumed run reuses that window.
start_date: str | None = None
end_date: str | None = None
def client_config(config: SimSourceConfig, api_version: str) -> ClientConfig:
return {
"base_url": f"https://www.sim.ai/api/{api_version}/",
"auth": {"type": "api_key", "name": "X-API-Key", "api_key": config.api_key, "location": "header"},
"headers": {"Accept": "application/json"},
"request_timeout": (10, 60),
"allowed_hosts": [],
"allow_redirects": False,
}
def endpoint_filters(config: SimSourceConfig, endpoint: SimEndpoint) -> dict[str, Any]:
params: dict[str, Any] = {}
if endpoint.workspace_scoped:
params["workspaceId"] = config.workspace_id
if endpoint.sort_field:
params["sortBy"] = endpoint.sort_field
params["sortOrder"] = "asc"
return params
def get_endpoint(config: SimSourceConfig, name: str, page_size: int | None = None) -> Endpoint:
endpoint = schema_for_resource(ALL_ENDPOINTS, name)
return {
"path": endpoint.path,
"params": {"limit": page_size or endpoint.page_size, **endpoint_filters(config, endpoint)},
**PAGE_EXTRA,
}
def get_resource(config: SimSourceConfig, name: str) -> EndpointResource:
return {
"name": name,
"table_format": "delta",
"write_disposition": "replace",
"endpoint": get_endpoint(config, name),
}
@frozen
class UsageWindow:
start: str
end: str
def usage_window(inputs: SourceInputs, resume: SimResumeConfig | None) -> UsageWindow:
if resume is not None and resume.start_date and resume.end_date:
return UsageWindow(start=resume.start_date, end=resume.end_date)
end = datetime.now(UTC).date()
start = end - timedelta(days=USAGE_WINDOW_DAYS)
last_value = parse_datetime_value(inputs.db_incremental_field_last_value)
if inputs.should_use_incremental_field and last_value is not None:
if last_value.date() < start:
inputs.logger.warning(
"Sim usage watermark is older than the longest window Sim serves; older events are skipped",
watermark=last_value.isoformat(),
window_start=start.isoformat(),
)
start = min(max(start, last_value.date()), end)
return UsageWindow(start=start.isoformat(), end=end.isoformat())
def _check_usage_access(
client: RESTClient, endpoint: SimEndpoint, organization_pages: Iterator[Any]
) -> tuple[bool, str | None]:
# Listing organizations succeeds for any key, but usage needs organization admin rights in at least one of them.
end = datetime.now(UTC).date()
start = end - timedelta(days=1)
found_organization = False
for page in organization_pages:
for organization in page:
if not organization.get("id"):
continue
found_organization = True
try:
next(
client.paginate(
path=endpoint.path.format(organization_id=organization["id"]),
params={
"limit": 1,
"preset": "custom",
"startDate": start.isoformat(),
"endDate": end.isoformat(),
},
data_selector="data",
data_selector_required=True,
)
)
except HTTPError as error:
if error.response is not None and error.response.status_code in (403, 404):
continue
raise
return True, None
# A key without organizations has no usage to read, so there is nothing to reject.
return (False, USAGE_PERMISSION_ERROR) if found_organization else (True, None)
def validate_credentials(
config: SimSourceConfig, api_version: str, schema_name: str | None = None
) -> tuple[bool, str | None]:
endpoint = schema_for_resource(ENDPOINTS, schema_name or "workflows")
# A fan-out table is readable when its parent list is, because the parent ids drive every child request.
probe_name = endpoint.fanout.parent_name if endpoint.fanout else endpoint.name
is_usage = endpoint.name == "usage_events"
probe = get_endpoint(config, probe_name, page_size=USAGE_PROBE_ORGANIZATIONS if is_usage else 1)
client_settings = client_config(config, api_version)
client = RESTClient(
base_url=client_settings["base_url"],
auth=create_auth(client_settings["auth"]),
headers=client_settings["headers"],
request_timeout=client_settings["request_timeout"],
allowed_hosts=[],
allow_redirects=False,
)
try:
pages = client.paginate(
path=cast(str, probe["path"]),
params=probe["params"],
paginator=JSONResponseCursorPaginator(cursor_path="nextCursor", cursor_param="cursor")
if is_usage
else None,
data_selector="data",
data_selector_required=True,
)
if is_usage:
return _check_usage_access(client, endpoint, pages)
next(pages)
except HTTPError as error:
status = error.response.status_code if error.response is not None else None
if status == 401:
return False, AUTH_ERROR
if status == 403:
if schema_name is None:
return True, None
return False, USAGE_PERMISSION_ERROR if endpoint.name == "usage_events" else PERMISSION_ERROR
if status == 404:
return False, WORKSPACE_ERROR
raise
finally:
client.session.close()
return True, None
def sim_source(
config: SimSourceConfig,
inputs: SourceInputs,
manager: ResumableSourceManager[SimResumeConfig],
api_version: str,
) -> SourceResponse:
endpoint = schema_for_resource(ENDPOINTS, inputs.schema_name)
resume = manager.load_state() if manager.can_resume() else None
window: UsageWindow | None = None
resource: Iterable[Any]
if endpoint.fanout is None:
def save_checkpoint(state: dict[str, Any] | None) -> None:
if state and state.get("cursor"):
manager.save_state(SimResumeConfig(cursor=str(state["cursor"])))
resource_config: RESTAPIConfig = {
"client": client_config(config, api_version),
"resources": [get_resource(config, inputs.schema_name)],
}
resource = rest_api_resource(
resource_config,
inputs.team_id,
inputs.job_id,
None,
resume_hook=save_checkpoint,
initial_paginator_state={"cursor": resume.cursor} if resume is not None and resume.cursor else None,
)
else:
child_params = endpoint_filters(config, endpoint)
if endpoint.name == "usage_events":
window = usage_window(inputs, resume)
child_params.update(preset="custom", startDate=window.start, endDate=window.end)
def save_fanout_checkpoint(state: dict[str, Any] | None) -> None:
manager.save_state(
SimResumeConfig(
fanout_state=state,
start_date=window.start if window else None,
end_date=window.end if window else None,
)
)
parent = ALL_ENDPOINTS[endpoint.fanout.parent_name]
resource = build_dependent_resource(
endpoint_configs=ALL_ENDPOINTS,
child_endpoint=endpoint.name,
fanout=dataclasses.replace(
endpoint.fanout, parent_params=endpoint_filters(config, parent), child_params=child_params
),
client_config=client_config(config, api_version),
path_format_values={},
team_id=inputs.team_id,
job_id=inputs.job_id,
db_incremental_field_last_value=None,
parent_endpoint_extra=PAGE_EXTRA,
child_endpoint_extra=PAGE_EXTRA,
resume_hook=save_fanout_checkpoint,
initial_paginator_state=resume.fanout_state if resume is not None else None,
)
return SourceResponse(
name=inputs.schema_name,
items=lambda: resource,
primary_keys=list(endpoint.primary_keys),
partition_keys=[endpoint.partition_key],
partition_mode="datetime",
partition_format="month",
sort_mode="asc",
)