Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add attio activities table (#126934)

Tom Owersa44fe443a829
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 52a9a36e93ee..db58865432c9 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -263,7 +263,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/asknicely/asknicely.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/assemblyai/assemblyai.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/assemblyai/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/attio/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/automox/automox.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/automox/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/autumn/autumn.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index ac3b581c310f..ca09f9677603 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -983,6 +983,16 @@ No material gaps found.
Note: This is a webhook-ingestion source (the directory contains webhook_template.py), and the 8 tables map exactly 1:1 onto the 8 event types Attentive's createwebhook endpoint accepts: sms.subscribed, sms.sent, sms.message_link_click, email.subscribed, email.unsubscribed, email.message_link_click, email.opened, custom_attribute.set. Full coverage of the webhook surface. Attentive's REST/GraphQL APIs are almost entirely write/ingest (POST events, attributes, subscriptions, catalog uploads); the only GETs are single-user lookups by phone/email, webhook config, catalog upload status and /me - no analytical collections to backfill from, so historical data before webhook setup is simply unavailable from the vendor.
+## Attio — gaps
+
+Today (10): `activities`, `companies`, `deals`, `lists`, `notes`, `people`, `tasks`, `users`, `workspace_members`, `workspaces`
+
+Diffed against: <https://docs.attio.com/changelog/rest-api>
+
+- [x] `Activities (GET /v2/activities)` — activity type definitions, so activity records can be labelled by type (low)
+
+Note: The main gap list for Attio is in COVERAGE_GAPS.md. `GET /v2/emails` was not added: it is alpha, Attio enables it per workspace on request, and it only returns emails filtered by linked records, participants, or domain, so it cannot list a workspace's emails.
+
## Auth0 — gaps
Today (13): `actions`, `clients`, `connections`, `log_streams`, `logs`, `organization_member_roles`, `organization_members`, `organizations`, `resource_servers`, `roles`, `user_roles`, `user_sessions`, `users`
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/attio.py b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/attio.py
index 6c7e77f2ef4f..d002d7513817 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/attio.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/attio.py
@@ -10,7 +10,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
RESTAPIConfig,
rest_api_resource,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ BasePaginator,
+ JSONResponseCursorPaginator,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
Endpoint,
EndpointResource,
@@ -21,9 +24,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
@frozen
class AttioResumeConfig:
- # Row offset of the next page to fetch. The Redis key is per job, so one job never reads
- # another endpoint's offset.
- offset: int
+ # Position of the next page to fetch: a row offset, or an Attio cursor for cursor-paginated
+ # endpoints. The Redis key is per job, so one job never reads another endpoint's position.
+ offset: int | None = None
+ cursor: str | None = None
class AttioOffsetPaginator(BasePaginator):
@@ -131,6 +135,11 @@ def get_resource(name: str) -> EndpointResource:
endpoint_config["paginator"] = AttioOffsetPaginator(
limit=config.page_size, use_json_body=True, initial_json=json_body
)
+ elif config.pagination == "cursor":
+ endpoint_config["params"] = {"limit": config.page_size}
+ endpoint_config["paginator"] = JSONResponseCursorPaginator(
+ cursor_path="pagination.next_cursor", cursor_param="cursor"
+ )
else:
endpoint_config["paginator"] = AttioOffsetPaginator(limit=config.page_size)
@@ -220,12 +229,16 @@ def attio_source(
if resumable_source_manager.can_resume():
resume = resumable_source_manager.load_state()
if resume is not None:
- initial_paginator_state = {"offset": resume.offset}
+ initial_paginator_state = {"offset": resume.offset, "cursor": resume.cursor}
def save_checkpoint(state: Optional[dict[str, Any]]) -> None:
# The offset points at the page after the one that carries this state, so a resume never
# skips a page that the pipeline has not written.
- if state and state.get("offset") is not None:
+ if not state:
+ return
+ if state.get("cursor") is not None:
+ resumable_source_manager.save_state(AttioResumeConfig(cursor=str(state["cursor"])))
+ elif state.get("offset") is not None:
resumable_source_manager.save_state(AttioResumeConfig(offset=int(state["offset"])))
resource = rest_api_resource(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/canonical_descriptions.py
index ef0e9e0f149d..4032b1e00ceb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/canonical_descriptions.py
@@ -104,4 +104,18 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"created_at": "Time at which the member was added to the workspace.",
},
},
+ "activities": {
+ "description": "An activity type defined in the Attio workspace, such as a call or site visit. Lists the type definitions, not the individual activity records.",
+ "docs_url": "https://docs.attio.com/rest-api/endpoint-reference/activities/list-activities",
+ "columns": {
+ "id": "Composite identifier for the activity type (workspace and activity id).",
+ "activity_id": "Unique identifier for the activity type.",
+ "api_slug": "URL-safe slug used to reference the activity type in the API.",
+ "singular_noun": "Singular display name of the activity type.",
+ "plural_noun": "Plural display name of the activity type.",
+ "extends": "The standard schemas this activity type extends.",
+ "is_system_activity": "Whether Attio defines the activity type, as opposed to a workspace user.",
+ "created_at": "Time at which the activity type was created.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/settings.py
index bedecc834251..c8bf308c5303 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/settings.py
@@ -1,8 +1,9 @@
-from dataclasses import dataclass
from typing import Literal
+from posthog.dataclasses import frozen
-@dataclass
+
+@frozen
class AttioEndpointConfig:
name: str
path: str
@@ -10,6 +11,7 @@ class AttioEndpointConfig:
partition_key: str = "created_at"
method: Literal["GET", "POST"] = "GET"
page_size: int = 500
+ pagination: Literal["offset", "cursor"] = "offset"
# Attio API doesn't support updatedAt filtering, so only full refresh is supported
@@ -69,6 +71,13 @@ ATTIO_ENDPOINTS: dict[str, AttioEndpointConfig] = {
method="GET",
primary_key="workspace_member_id",
),
+ "activities": AttioEndpointConfig(
+ name="activities",
+ path="/v2/activities",
+ method="GET",
+ primary_key="activity_id",
+ pagination="cursor",
+ ),
}
ENDPOINTS = tuple(ATTIO_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/source.py
index ef10319b52f9..d15dc10d40b2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/source.py
@@ -74,6 +74,7 @@ You can generate an API key in your Attio workspace settings. Check out [this gu
- `note:read` - To read notes
- `task:read` - To read tasks
- `user_management:read` - To read workspace members
+- `activity_configuration:read` - To read activity types
""",
iconPath="/static/services/attio.png",
docsUrl="https://posthog.com/docs/cdp/sources/attio",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/tests/test_attio.py b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/tests/test_attio.py
index 6ed2e29d99e2..685576b0661e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/tests/test_attio.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/tests/test_attio.py
@@ -88,3 +88,54 @@ class TestAttioResume:
assert [request.json() for request in result.requests] == [
{"sorts": [{"attribute": "created_at", "direction": "asc"}], "offset": 500, "limit": 500}
]
+
+
+def _cursor_page(start: int, count: int, next_cursor: str | None) -> ScriptedResponse:
+ rows = [
+ {"id": {"workspace_id": "ws", "activity_id": f"row-{i}"}, "created_at": "2024-01-01"}
+ for i in range(start, start + count)
+ ]
+ return ScriptedResponse(json={"data": rows, "pagination": {"next_cursor": next_cursor}})
+
+
+class TestAttioCursorPagination:
+ @staticmethod
+ def _driver() -> SourceDriver:
+ return SourceDriver(AttioSource(), AttioSourceConfig(api_key="key"))
+
+ @pytest.mark.parametrize(
+ "resume_state, responses, expected_cursors, expected_rows, expected_states",
+ [
+ (
+ None,
+ [_cursor_page(0, 2, "c1"), _cursor_page(2, 2, "c2"), _cursor_page(4, 1, None)],
+ [None, "c1", "c2"],
+ 5,
+ [AttioResumeConfig(cursor="c1"), AttioResumeConfig(cursor="c2")],
+ ),
+ (
+ AttioResumeConfig(cursor="c2"),
+ [_cursor_page(4, 2, "c3"), _cursor_page(6, 1, None)],
+ ["c2", "c3"],
+ 3,
+ [AttioResumeConfig(cursor="c3")],
+ ),
+ ],
+ )
+ def test_activities_follow_next_cursor_and_stage_it(
+ self,
+ resume_state: AttioResumeConfig | None,
+ responses: list[ScriptedResponse],
+ expected_cursors: list[str | None],
+ expected_rows: int,
+ expected_states: list[AttioResumeConfig],
+ ) -> None:
+ result = self._driver().run("activities", responses, resume_state=resume_state)
+
+ assert result.raised is None
+ assert [request.param("cursor") for request in result.requests] == expected_cursors
+ assert all(request.param("limit") == "500" for request in result.requests)
+ assert all(request.param("offset") is None for request in result.requests)
+ assert len(result.rows) == expected_rows
+ assert "activity_id" in result.rows[0]
+ assert result.saved_states == expected_states