Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

8.1 KB
Raw Blame
from dataclasses import dataclass, field
from typing import Literal, Optional

from products.warehouse_sources.sources.sdk import IncrementalField

# Twilio Segment exposes two regional hosts for the same workspace-scoped Public API. The token is
# bound to one region, so the user picks it at connect time (Airbyte models this as `region: api|eu1`).
REGION_BASE_URLS: dict[str, str] = {
    "api": "https://api.segmentapis.com",
    "eu1": "https://eu1.api.segmentapis.com",
}
DEFAULT_REGION = "api"


@dataclass(frozen=False)
class SegmentEndpointConfig:
    name: str
    path: str
    # Keys must be unique table-wide. Most resources expose a global `id`; labels have none, so they
    # use the natural (key, value) composite.
    primary_keys: list[str] = field(default_factory=lambda: ["id"])
    # `GET /` (Get Workspace) returns a single object under `data.<object_key>` with no pagination,
    # rather than a paginated list. Such endpoints yield exactly one row.
    single_object_key: Optional[str] = None
    # A stable, immutable datetime field to partition on. Segment's config/admin objects carry no
    # timestamps, so only audit events, sync history and the time-series usage tables set this.
    partition_key: Optional[str] = None
    incremental_fields: list[IncrementalField] = field(default_factory=list)
    should_sync_default: bool = True
    # Top-level fields stripped from every row before it's persisted. Segment's config endpoints
    # embed credential-like blobs — destination/warehouse connection `settings` (third-party API
    # keys, DB passwords) and source `writeKeys`. These are free-form per integration, so there's
    # no reliable allowlist of safe sub-keys; we drop the whole field rather than risk leaking
    # secrets into a queryable warehouse table.
    redacted_fields: frozenset[str] = field(default_factory=frozenset)
    # Child resources read once per parent row. `path` then holds the parent placeholders, and
    # `segment.py` lists the parents and resumes per parent.
    fanout: Optional[
        Literal[
            "tracking_plan_rules",
            "delivery_metrics",
            "reverse_etl_syncs",
            "event_volume",
            "mtu_usage",
            "warehouse_syncs",
            "delivery_overview",
        ]
    ] = None


# The Segment Public API is a workspace configuration/admin/usage API (not the event/Profile data
# plane). Every listing endpoint uses cursor pagination (`pagination[count]` / `pagination[cursor]`)
# and wraps its rows in `{"data": {"<resource>": [...], "pagination": {...}}}`.
#
# These resources expose no server-side timestamp filter, so they are full refresh only. Audit events
# document `startTime`/`endTime` filters and so are a future incremental candidate, but that needs a
# live-token smoke test (future-date cutoff) to confirm the filter is honored before we enable it.
SEGMENT_ENDPOINTS: dict[str, SegmentEndpointConfig] = {
    "workspace": SegmentEndpointConfig(
        name="workspace",
        path="/",
        single_object_key="workspace",
    ),
    "sources": SegmentEndpointConfig(name="sources", path="/sources", redacted_fields=frozenset({"writeKeys"})),
    "destinations": SegmentEndpointConfig(
        name="destinations", path="/destinations", redacted_fields=frozenset({"settings"})
    ),
    "warehouses": SegmentEndpointConfig(name="warehouses", path="/warehouses", redacted_fields=frozenset({"settings"})),
    "tracking_plans": SegmentEndpointConfig(name="tracking_plans", path="/tracking-plans"),
    "transformations": SegmentEndpointConfig(name="transformations", path="/transformations"),
    "reverse_etl_models": SegmentEndpointConfig(name="reverse_etl_models", path="/reverse-etl-models"),
    "iam_users": SegmentEndpointConfig(name="iam_users", path="/users"),
    "iam_groups": SegmentEndpointConfig(name="iam_groups", path="/groups"),
    "labels": SegmentEndpointConfig(name="labels", path="/labels", primary_keys=["key", "value"]),
    "audit_events": SegmentEndpointConfig(name="audit_events", path="/audit-events", partition_key="timestamp"),
    # Segment-wide catalogs that resolve the `metadata.id` carried on sources, destinations and warehouses.
    "destinations_catalog": SegmentEndpointConfig(name="destinations_catalog", path="/catalog/destinations"),
    "sources_catalog": SegmentEndpointConfig(name="sources_catalog", path="/catalog/sources"),
    "warehouses_catalog": SegmentEndpointConfig(name="warehouses_catalog", path="/catalog/warehouses"),
    # Rules have no id. A plan holds one rule per (type, key, version), and keyless rule types
    # (identify, group, common) get an empty `key`.
    "tracking_plan_rules": SegmentEndpointConfig(
        name="tracking_plan_rules",
        path="/tracking-plans/{tracking_plan_id}/rules",
        primary_keys=["trackingPlanId", "type", "key", "version"],
        fanout="tracking_plan_rules",
    ),
    # One row per destination and metric, with totals over the last DELIVERY_METRICS_WINDOW_DAYS full days.
    "destination_delivery_metrics": SegmentEndpointConfig(
        name="destination_delivery_metrics",
        path="/destinations/{destination_id}/delivery-metrics",
        primary_keys=["destinationId", "metricName"],
        fanout="delivery_metrics",
    ),
    # A sync can load into several subscriptions of one model, so the sync id is unique only per subscription.
    "reverse_etl_syncs": SegmentEndpointConfig(
        name="reverse_etl_syncs",
        path="/reverse-etl-models/{model_id}/subscriptionId/{subscription_id}/syncs",
        primary_keys=["subscriptionId", "syncId"],
        partition_key="startedAt",
        fanout="reverse_etl_syncs",
    ),
    # One row per source and day over the last METRICS_SERIES_WINDOW_DAYS full days.
    "event_volume": SegmentEndpointConfig(
        name="event_volume",
        path="/events/volume",
        primary_keys=["sourceId", "time"],
        partition_key="time",
        fanout="event_volume",
    ),
    # Daily cumulative MTU snapshots for the current and the previous usage month.
    "workspace_mtu_usage": SegmentEndpointConfig(
        name="workspace_mtu_usage",
        path="/usage/mtu/daily",
        primary_keys=["periodStart", "timestamp"],
        partition_key="timestamp",
        fanout="mtu_usage",
    ),
    "source_mtu_usage": SegmentEndpointConfig(
        name="source_mtu_usage",
        path="/usage/mtu/sources/daily",
        primary_keys=["sourceId", "periodStart", "timestamp"],
        partition_key="timestamp",
        fanout="mtu_usage",
    ),
    # Sync reports have no id. One warehouse loads each of its sources once per start time.
    "warehouse_syncs": SegmentEndpointConfig(
        name="warehouse_syncs",
        path="/warehouses/{warehouse_id}/syncs",
        primary_keys=["warehouseId", "sourceId", "start"],
        partition_key="start",
        fanout="warehouse_syncs",
    ),
    # One row per pipeline stage, source, destination and day. Source-side stages have an empty destinationId.
    "delivery_overview": SegmentEndpointConfig(
        name="delivery_overview",
        path="/delivery-overview/{stage}",
        primary_keys=["stage", "sourceId", "destinationId", "time"],
        partition_key="time",
        fanout="delivery_overview",
    ),
}

# Segment accepts a DAY-granularity range of at most 14 days that starts at most 14 days ago. 12 whole
# days stay inside both limits, also for a resumed run that keeps the window of its first attempt.
DELIVERY_METRICS_WINDOW_DAYS = 12
DELIVERY_METRICS_MAX_LOOKBACK_DAYS = 14

# Delivery Overview accepts a DAY-granularity range of at most 30 days that starts at most 30 days ago.
# Events volume documents no limit, so it uses the same window.
METRICS_SERIES_WINDOW_DAYS = 28
METRICS_SERIES_MAX_LOOKBACK_DAYS = 30

# Stages of the Delivery Overview funnel, by the side of the pipeline that reports them.
DELIVERY_OVERVIEW_SOURCE_STAGES = ("successfully-received", "failed-on-ingest", "filtered-at-source")
DELIVERY_OVERVIEW_DESTINATION_STAGES = ("filtered-at-destination", "failed-delivery", "successful-delivery")

ENDPOINTS = tuple(SEGMENT_ENDPOINTS.keys())

INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
    name: config.incremental_fields for name, config in SEGMENT_ENDPOINTS.items()
}