Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

18.7 KB
Raw Blame
from dataclasses import field
from typing import Literal

from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType, PartitionFormat, frozen

PaginatorKind = Literal["single", "cursor", "next_url", "pages", "page_number", "search", "substream", "scroll"]
PartitionMode = Literal["md5", "datetime"]
HttpMethod = Literal["GET", "POST"]
SortMode = Literal["asc", "desc"]


@frozen
class IntercomEndpointConfig:
    name: str
    path: str
    data_selector: str
    paginator_kind: PaginatorKind
    method: HttpMethod = "GET"
    primary_keys: list[str] = field(default_factory=lambda: ["id"])
    partition_key: str = "created_at"
    partition_mode: PartitionMode = "datetime"
    partition_format: PartitionFormat | None = "week"
    partition_count: int = 1
    partition_size: int = 1
    page_size: int = 150
    # Where page-number endpoints report their page count: top level, or nested under `pages`.
    total_pages_path: str = "total_pages"
    sort_mode: SortMode | None = "asc"
    extra_params: dict[str, str] = field(default_factory=dict)
    # Query-string param Intercom expects for the cursor filter (e.g. "created_at_after"
    # on /admins/activity_logs). Only set on endpoints that filter via query params.
    incremental_query_param: str | None = None
    # Substream wiring — when set, this endpoint is fetched per-row of `parent_endpoint`.
    parent_endpoint: str | None = None
    # Intercom versions that serve this endpoint. `None` means every version the source supports.
    api_versions: tuple[str, ...] | None = None


# Endpoint contracts validated against the live Intercom REST API (version 2.13).
# Selectors, pagination shape, and partition keys reflect what the API actually
# returns — not the docs.
# The endpoints with `api_versions` set exist only from 2.16 or 2.17 on and follow
# Intercom's published OpenAPI description for that version.
#
# Real-incremental endpoints (server-side filter honored): contacts, conversations,
# tickets, activity_logs, macros, conversation_parts (via parent). Everything else is
# full-refresh because Intercom either has no server filter (companies, segments,
# company_segments — Airbyte/Stitch sync these "incrementally" but actually walk
# every page each run) or no timestamp at all (data_attributes).
INTERCOM_ENDPOINTS: dict[str, IntercomEndpointConfig] = {
    "admins": IntercomEndpointConfig(
        name="admins",
        path="/admins",
        data_selector="admins",
        paginator_kind="single",
        partition_key="id",
        partition_mode="md5",
        partition_format=None,
    ),
    "teams": IntercomEndpointConfig(
        name="teams",
        path="/teams",
        data_selector="teams",
        paginator_kind="single",
        partition_key="id",
        partition_mode="md5",
        partition_format=None,
    ),
    "tags": IntercomEndpointConfig(
        name="tags",
        path="/tags",
        data_selector="data",
        paginator_kind="single",
        partition_key="id",
        partition_mode="md5",
        partition_format=None,
    ),
    "segments": IntercomEndpointConfig(
        name="segments",
        path="/segments",
        data_selector="segments",
        paginator_kind="single",
    ),
    "companies": IntercomEndpointConfig(
        # `GET /companies/scroll` (the Scroll API), not `POST /companies/list`.
        # The list endpoint is hard-capped at 10,000 companies — paging past it
        # returns `400 bad_request: page limit reached, please use scroll API`
        # (60 per page × 167 pages ≈ the 10k ceiling). The Scroll API has no such
        # ceiling. It's walked by `_iter_companies` (see intercom.py), routed via
        # the `scroll` paginator_kind rather than the `RESTAPIConfig` path because
        # the cursor lives in a `scroll_param` the framework paginators don't model.
        # Intercom allows only one open scroll per workspace; a stale scroll left
        # by an interrupted or concurrent sync is handled by `_open_companies_scroll`'s
        # `scroll_exists` backoff.
        name="companies",
        path="/companies/scroll",
        data_selector="data",
        paginator_kind="scroll",
    ),
    "contacts": IntercomEndpointConfig(
        name="contacts",
        path="/contacts/search",
        data_selector="data",
        paginator_kind="search",
        method="POST",
        partition_key="created_at",
    ),
    "conversations": IntercomEndpointConfig(
        name="conversations",
        path="/conversations/search",
        data_selector="conversations",
        paginator_kind="search",
        method="POST",
        partition_key="created_at",
    ),
    "tickets": IntercomEndpointConfig(
        # `POST /tickets/search` mirrors contacts/conversations: it accepts
        # the same `{"field":"updated_at","operator":">","value":<ts>}` body
        # filter and the same cursor pagination shape.
        name="tickets",
        path="/tickets/search",
        data_selector="tickets",
        paginator_kind="search",
        method="POST",
        partition_key="created_at",
    ),
    "articles": IntercomEndpointConfig(
        name="articles",
        path="/articles",
        data_selector="data",
        paginator_kind="next_url",
    ),
    "activity_logs": IntercomEndpointConfig(
        # `GET /admins/activity_logs?created_at_after=<unix_ts>` is honored
        # server-side (verified with future-date probe). The endpoint sorts
        # descending by `created_at` and ignores any `?sort=` param, so
        # SourceResponse declares `sort_mode="desc"` to match.
        name="activity_logs",
        path="/admins/activity_logs",
        data_selector="activity_logs",
        paginator_kind="next_url",
        partition_key="created_at",
        sort_mode="desc",
        incremental_query_param="created_at_after",
    ),
    "company_attributes": IntercomEndpointConfig(
        # `GET /data_attributes?model=company` — built-in plus custom
        # company-level attributes. No `id`/timestamp on rows; primary key
        # is `name` (unique within a model).
        name="company_attributes",
        path="/data_attributes",
        data_selector="data",
        paginator_kind="single",
        primary_keys=["name"],
        partition_key="name",
        partition_mode="md5",
        partition_format=None,
        extra_params={"model": "company"},
    ),
    "contact_attributes": IntercomEndpointConfig(
        name="contact_attributes",
        path="/data_attributes",
        data_selector="data",
        paginator_kind="single",
        primary_keys=["name"],
        partition_key="name",
        partition_mode="md5",
        partition_format=None,
        extra_params={"model": "contact"},
    ),
    "conversation_parts": IntercomEndpointConfig(
        # Substream of `conversations`: walk the same `POST /conversations/search`
        # query (so the parent's `updated_at >` filter is server-honored), then
        # fetch `GET /conversations/{id}` for each parent and yield each part.
        # Effectively real-incremental — only conversations whose `updated_at`
        # advanced are refetched. Each yielded part carries `conversation_id`.
        name="conversation_parts",
        path="",
        data_selector="",
        paginator_kind="substream",
        parent_endpoint="conversations",
        partition_key="created_at",
        # The parts of each conversation are a separate series, so the rows of a run have no single order.
        sort_mode=None,
    ),
    "help_centers": IntercomEndpointConfig(
        # `GET /help_center/help_centers` — one row per Help Center site. A workspace
        # has a handful at most and the response carries no `pages` block, so it's a
        # single-page read. Decodes `collections.help_center_id`.
        name="help_centers",
        path="/help_center/help_centers",
        data_selector="data",
        paginator_kind="single",
        partition_key="id",
        partition_mode="md5",
        partition_format=None,
    ),
    "collections": IntercomEndpointConfig(
        # `GET /help_center/collections` — the Help Center structure behind the
        # `articles` table (articles carry `parent_id`, collections carry
        # `parent_id`/`help_center_id`). Intercom documents the order as descending
        # on `updated_at` and offers no server-side timestamp filter, so this is
        # full refresh.
        name="collections",
        path="/help_center/collections",
        data_selector="data",
        paginator_kind="pages",
        sort_mode="desc",
    ),
    "ticket_types": IntercomEndpointConfig(
        # `GET /ticket_types` — decodes the `ticket_type` id carried on `tickets`.
        # Workspace-level config list with no `pages` block.
        name="ticket_types",
        path="/ticket_types",
        data_selector="data",
        paginator_kind="single",
        partition_key="id",
        partition_mode="md5",
        partition_format=None,
    ),
    "ticket_states": IntercomEndpointConfig(
        # `GET /ticket_states` — decodes the ticket state id carried on `tickets`,
        # and carries the customer-facing vs internal labels for each state.
        name="ticket_states",
        path="/ticket_states",
        data_selector="data",
        paginator_kind="single",
        partition_key="id",
        partition_mode="md5",
        partition_format=None,
    ),
    "subscription_types": IntercomEndpointConfig(
        # `GET /subscription_types` — the consent/subscription types a contact can
        # opt in or out of. Rows have no timestamps, so partition on the id.
        name="subscription_types",
        path="/subscription_types",
        data_selector="data",
        paginator_kind="single",
        partition_key="id",
        partition_mode="md5",
        partition_format=None,
    ),
    "news_items": IntercomEndpointConfig(
        # `GET /news/news_items` — product announcements published to newsfeeds.
        # No server-side timestamp filter, so full refresh.
        name="news_items",
        path="/news/news_items",
        data_selector="data",
        paginator_kind="pages",
    ),
    "newsfeeds": IntercomEndpointConfig(
        # `GET /news/newsfeeds` — the feeds news items are assigned to. A workspace
        # has very few, so partition on the id rather than by week.
        name="newsfeeds",
        path="/news/newsfeeds",
        data_selector="data",
        paginator_kind="pages",
        partition_key="id",
        partition_mode="md5",
        partition_format=None,
    ),
    "company_segments": IntercomEndpointConfig(
        # Substream of `companies`. Intercom doesn't expose a server-side
        # filter on either parent or child, so this is full-refresh (matches
        # the new "fake incremental → full refresh" rule). Each yielded
        # segment carries `company_id`.
        name="company_segments",
        path="",
        data_selector="",
        paginator_kind="substream",
        parent_endpoint="companies",
        primary_keys=["company_id", "id"],
        partition_key="id",
        partition_mode="md5",
        partition_format=None,
    ),
    "conversation_attributes": IntercomEndpointConfig(
        # `GET /conversations/attributes?include_archived=true` — the conversation data
        # attribute definitions, with list options and relationship references. Archived
        # attributes are included because older conversations still carry their values.
        name="conversation_attributes",
        path="/conversations/attributes",
        data_selector="data",
        paginator_kind="single",
        partition_key="id",
        partition_mode="md5",
        partition_format=None,
        extra_params={"include_archived": "true"},
        api_versions=("2.16", "2.17"),
    ),
    "macros": IntercomEndpointConfig(
        # `GET /macros` — saved replies. Paged by a `pages.next.starting_after` cursor,
        # sorted descending on `updated_at`, and filtered server-side by `updated_since`
        # (Unix seconds), so `sort_mode="desc"` like `activity_logs`. Unlike most
        # Intercom objects, `created_at`/`updated_at` are ISO 8601 strings here.
        name="macros",
        path="/macros",
        data_selector="data",
        paginator_kind="pages",
        sort_mode="desc",
        incremental_query_param="updated_since",
        api_versions=("2.16", "2.17"),
    ),
    "audiences": IntercomEndpointConfig(
        # `GET /audiences` — saved audience definitions (targeting predicates). Paged by
        # `page`/`per_page` (max 50) with a top-level `total_pages`, no timestamp filter.
        name="audiences",
        path="/audiences",
        data_selector="data",
        paginator_kind="page_number",
        page_size=50,
        api_versions=("2.16", "2.17"),
    ),
    "content_snippets": IntercomEndpointConfig(
        # `GET /content_snippets` — snippets that feed Fin and Copilot answers. Same
        # `page`/`per_page` (max 50) shape as `audiences`, no timestamp filter.
        name="content_snippets",
        path="/content_snippets",
        data_selector="data",
        paginator_kind="page_number",
        page_size=50,
        api_versions=("2.16", "2.17"),
    ),
    "contact_activities": IntercomEndpointConfig(
        # `GET /contacts/activities` — workspace feed of contact lifecycle events (created,
        # role updated, merged, deleted). Oldest first, paged by a `pages.next.starting_after`
        # cursor, with no timestamp filter, so full refresh.
        name="contact_activities",
        path="/contacts/activities",
        data_selector="data",
        paginator_kind="pages",
        api_versions=("2.17",),
    ),
    "deleted_contacts": IntercomEndpointConfig(
        # `GET /contacts/deleted` — contacts that are archived, blocked, merged, or deleted.
        # Rows carry only `updated_at`, which changes, so partition on the id.
        name="deleted_contacts",
        path="/contacts/deleted",
        data_selector="data",
        paginator_kind="page_number",
        partition_key="id",
        partition_mode="md5",
        partition_format=None,
        total_pages_path="pages.total_pages",
        api_versions=("2.17",),
    ),
    "folders": IntercomEndpointConfig(
        # `GET /folders` — the folders articles and other content are organized into
        # (decodes `articles.folder_id`). Intercom documents no `per_page` maximum, so this
        # uses the 50 its examples show.
        name="folders",
        path="/folders",
        data_selector="data",
        paginator_kind="page_number",
        page_size=50,
        total_pages_path="pages.total_pages",
        api_versions=("2.17",),
    ),
    "dynamic_contents": IntercomEndpointConfig(
        # `GET /dynamic_contents` — reusable Help Center content with a variation per locale
        # and brand. Most recently updated first, `per_page` max 50, no timestamp filter.
        # Needs the `read_dynamic_content` scope and the feature enabled on the workspace.
        name="dynamic_contents",
        path="/dynamic_contents",
        data_selector="data",
        paginator_kind="page_number",
        page_size=50,
        sort_mode="desc",
        api_versions=("2.17",),
    ),
    "conversation_hitl_events": IntercomEndpointConfig(
        # Substream of `conversations`: `GET /conversations/{id}/hitl_events` for each parent
        # the conversations search returns. Events have no id of their own. Every event of one
        # consultation shares `hitl_cycle_id`, and each type occurs once per consultation or
        # once per feedback round, hence the composite key. Each yielded event carries
        # `conversation_id` and `round_index` (0 for event types outside a round).
        name="conversation_hitl_events",
        path="",
        data_selector="",
        paginator_kind="substream",
        parent_endpoint="conversations",
        primary_keys=["conversation_id", "hitl_cycle_id", "event_type", "round_index"],
        api_versions=("2.17",),
    ),
}

ENDPOINTS = tuple(INTERCOM_ENDPOINTS.keys())


# Endpoints that support incremental sync via a server-honored filter on the
# request itself — verified empirically against the live API:
# - contacts/conversations/tickets: POST /<resource>/search accepts
#   `{"field":"updated_at","operator":">","value":<unix_ts>}` in the body.
# - activity_logs: GET /admins/activity_logs?created_at_after=<unix_ts>.
# - macros: GET /macros?updated_since=<unix_ts>.
# - conversation_parts: child of conversations; the parent's `updated_at >`
#   filter is server-honored, so parents whose timestamp didn't advance
#   aren't refetched.
# - conversation_hitl_events: child of conversations, filtered on the parent's
#   `updated_at` like conversation_parts, but from a source cursor rather than the
#   rows' `created_at`. Intercom lists the events only once a consultation ends,
#   and the end (a reply, an escalation, a takeover) updates the conversation, so
#   a parent with new events is refetched.
#
# Companies, segments, and company_segments are intentionally excluded: their
# list endpoints don't accept a timestamp filter, and a "client-side cursor"
# that walks every page each run is identical in API cost to a full refresh.
#
# `field_type=Integer` because Intercom returns Unix epoch seconds, not ISO —
# except on macros, which return ISO 8601 strings.
INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
    "contacts": [
        {
            "label": "updated_at",
            "type": IncrementalFieldType.DateTime,
            "field": "updated_at",
            "field_type": IncrementalFieldType.Integer,
        }
    ],
    "conversations": [
        {
            "label": "updated_at",
            "type": IncrementalFieldType.DateTime,
            "field": "updated_at",
            "field_type": IncrementalFieldType.Integer,
        }
    ],
    "tickets": [
        {
            "label": "updated_at",
            "type": IncrementalFieldType.DateTime,
            "field": "updated_at",
            "field_type": IncrementalFieldType.Integer,
        }
    ],
    "activity_logs": [
        {
            "label": "created_at",
            "type": IncrementalFieldType.DateTime,
            "field": "created_at",
            "field_type": IncrementalFieldType.Integer,
        }
    ],
    "conversation_parts": [
        {
            "label": "updated_at",
            "type": IncrementalFieldType.DateTime,
            "field": "updated_at",
            "field_type": IncrementalFieldType.Integer,
        }
    ],
    "conversation_hitl_events": [
        {
            "label": "created_at",
            "type": IncrementalFieldType.DateTime,
            "field": "created_at",
            "field_type": IncrementalFieldType.Integer,
        }
    ],
    "macros": [
        {
            "label": "updated_at",
            "type": IncrementalFieldType.DateTime,
            "field": "updated_at",
            "field_type": IncrementalFieldType.DateTime,
        }
    ],
}