Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.2 KB
Raw Blame
from collections.abc import Callable, Iterator
from dataclasses import dataclass
from functools import partial
from typing import Any

from products.warehouse_sources.sources.schematic.settings import (
    PAGE_SIZE,
    SCHEMATIC_BASE_URL,
    SCHEMATIC_ENDPOINTS,
    SchematicEndpointConfig,
)
from products.warehouse_sources.sources.sdk import (
    ClientConfig,
    Resource,
    RESTAPIConfig,
    ResumableSourceManager,
    SourceResponse,
    make_tracked_session,
    rest_api_resource,
    validate_via_probe,
)

API_KEY_HEADER = "X-Schematic-Api-Key"


@dataclass(frozen=True)
class SchematicResumeConfig:
    offset: int
    # Which `is_active` pass the offset belongs to, for endpoints read in an active and an inactive pass.
    is_active: bool | None = None


def _client_config(api_key: str) -> ClientConfig:
    return {
        "base_url": SCHEMATIC_BASE_URL,
        "headers": {"Accept": "application/json"},
        "auth": {"type": "api_key", "api_key": api_key, "name": API_KEY_HEADER, "location": "header"},
        # `requests` keeps a custom header on a cross-origin redirect, so a redirect would replay the key.
        "allow_redirects": False,
        "allowed_hosts": [],
    }


def _resource(
    api_key: str,
    config: SchematicEndpointConfig,
    team_id: int,
    job_id: str,
    resumable_source_manager: ResumableSourceManager[SchematicResumeConfig],
    is_active: bool | None,
    initial_offset: int | None,
) -> Resource:
    params: dict[str, Any] = dict(config.extra_params)
    if is_active is not None:
        params["is_active"] = "true" if is_active else "false"
    rest_config: RESTAPIConfig = {
        "client": _client_config(api_key),
        "resource_defaults": {},
        "resources": [
            {
                "name": config.name,
                "table_name": config.name,
                "write_disposition": "replace",
                "table_format": "delta",
                "endpoint": {
                    "path": config.path,
                    "params": params,
                    # The response carries no total, so the walk ends on the first short page.
                    "data_selector": "data",
                    "data_selector_required": True,
                    "paginator": {"type": "offset", "limit": PAGE_SIZE, "total_path": None},
                },
            }
        ],
    }

    def save_checkpoint(state: dict[str, Any] | None) -> None:
        if state is not None:
            resumable_source_manager.save_state(SchematicResumeConfig(offset=int(state["offset"]), is_active=is_active))

    return rest_api_resource(
        rest_config,
        team_id,
        job_id,
        db_incremental_field_last_value=None,
        resume_hook=save_checkpoint,
        initial_paginator_state=None if initial_offset is None else {"offset": initial_offset},
    )


def _active_then_inactive(
    build: Callable[[bool, int | None], Resource],
    resumable_source_manager: ResumableSourceManager[SchematicResumeConfig],
    resume: SchematicResumeConfig | None,
) -> Iterator[Any]:
    if resume is None or resume.is_active is not False:
        yield from build(True, resume.offset if resume is not None else None)
        # Every active row is handed on, so a resume from here starts the inactive pass.
        resumable_source_manager.save_state(SchematicResumeConfig(offset=0, is_active=False))
        yield from build(False, None)
    else:
        yield from build(False, resume.offset)


def schematic_source(
    api_key: str,
    endpoint: str,
    team_id: int,
    job_id: str,
    resumable_source_manager: ResumableSourceManager[SchematicResumeConfig],
) -> SourceResponse:
    config = SCHEMATIC_ENDPOINTS[endpoint]
    resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None

    if config.include_inactive:
        build = partial(_resource, api_key, config, team_id, job_id, resumable_source_manager)
        first = build(True, None)
        # The chain hands each page on unchanged and holds no rows, so as a `Resource` it keeps the
        # framework's checkpoints.
        resource = Resource(
            partial(_active_then_inactive, build, resumable_source_manager, resume),
            name=endpoint,
            hints=first._hints,
        )
    else:
        resource = _resource(
            api_key,
            config,
            team_id,
            job_id,
            resumable_source_manager,
            None,
            resume.offset if resume is not None else None,
        )

    return SourceResponse(
        name=endpoint,
        items=lambda: resource,
        primary_keys=list(config.primary_keys),
        partition_count=1,
        partition_size=1,
        partition_mode="datetime" if config.partition_key else None,
        partition_format="month" if config.partition_key else None,
        partition_keys=[config.partition_key] if config.partition_key else None,
        column_hints=resource.column_hints,
    )


def validate_credentials(api_key: str) -> tuple[bool, int | None]:
    return validate_via_probe(
        lambda: make_tracked_session(redact_values=(api_key,)),
        f"{SCHEMATIC_BASE_URL}/companies?limit=1",
        headers={API_KEY_HEADER: api_key, "Accept": "application/json"},
        timeout=15,
        allow_redirects=False,
    )