:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add attio activities table (#126934)
Tom Owersa44fe443a829
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 52a9a36e93ee..db58865432c9 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -263,7 +263,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/asknicely/asknicely.py1 products/warehouse_sources/backend/temporal/data_imports/sources/assemblyai/assemblyai.py1 products/warehouse_sources/backend/temporal/data_imports/sources/assemblyai/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/attio/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/automox/automox.py1 products/warehouse_sources/backend/temporal/data_imports/sources/automox/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/autumn/autumn.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex ac3b581c310f..ca09f9677603 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -983,6 +983,16 @@ No material gaps found.Note: This is a webhook-ingestion source (the directory contains webhook_template.py), and the 8 tables map exactly 1:1 onto the 8 event types Attentive's createwebhook endpoint accepts: sms.subscribed, sms.sent, sms.message_link_click, email.subscribed, email.unsubscribed, email.message_link_click, email.opened, custom_attribute.set. Full coverage of the webhook surface. Attentive's REST/GraphQL APIs are almost entirely write/ingest (POST events, attributes, subscriptions, catalog uploads); the only GETs are single-user lookups by phone/email, webhook config, catalog upload status and /me - no analytical collections to backfill from, so historical data before webhook setup is simply unavailable from the vendor.+## Attio — gaps++Today (10): `activities`, `companies`, `deals`, `lists`, `notes`, `people`, `tasks`, `users`, `workspace_members`, `workspaces`++Diffed against: <https://docs.attio.com/changelog/rest-api>++- [x] `Activities (GET /v2/activities)` — activity type definitions, so activity records can be labelled by type (low)++Note: The main gap list for Attio is in COVERAGE_GAPS.md. `GET /v2/emails` was not added: it is alpha, Attio enables it per workspace on request, and it only returns emails filtered by linked records, participants, or domain, so it cannot list a workspace's emails.+## Auth0 — gapsToday (13): `actions`, `clients`, `connections`, `log_streams`, `logs`, `organization_member_roles`, `organization_members`, `organizations`, `resource_servers`, `roles`, `user_roles`, `user_sessions`, `users`diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/attio.py b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/attio.pyindex 6c7e77f2ef4f..d002d7513817 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/attio.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/attio.py@@ -10,7 +10,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resource,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (+ BasePaginator,+ JSONResponseCursorPaginator,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (Endpoint,EndpointResource,@@ -21,9 +24,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ@frozenclass AttioResumeConfig:- # Row offset of the next page to fetch. The Redis key is per job, so one job never reads- # another endpoint's offset.- offset: int+ # Position of the next page to fetch: a row offset, or an Attio cursor for cursor-paginated+ # endpoints. The Redis key is per job, so one job never reads another endpoint's position.+ offset: int | None = None+ cursor: str | None = Noneclass AttioOffsetPaginator(BasePaginator):@@ -131,6 +135,11 @@ def get_resource(name: str) -> EndpointResource:endpoint_config["paginator"] = AttioOffsetPaginator(limit=config.page_size, use_json_body=True, initial_json=json_body)+ elif config.pagination == "cursor":+ endpoint_config["params"] = {"limit": config.page_size}+ endpoint_config["paginator"] = JSONResponseCursorPaginator(+ cursor_path="pagination.next_cursor", cursor_param="cursor"+ )else:endpoint_config["paginator"] = AttioOffsetPaginator(limit=config.page_size)@@ -220,12 +229,16 @@ def attio_source(if resumable_source_manager.can_resume():resume = resumable_source_manager.load_state()if resume is not None:- initial_paginator_state = {"offset": resume.offset}+ initial_paginator_state = {"offset": resume.offset, "cursor": resume.cursor}def save_checkpoint(state: Optional[dict[str, Any]]) -> None:# The offset points at the page after the one that carries this state, so a resume never# skips a page that the pipeline has not written.- if state and state.get("offset") is not None:+ if not state:+ return+ if state.get("cursor") is not None:+ resumable_source_manager.save_state(AttioResumeConfig(cursor=str(state["cursor"])))+ elif state.get("offset") is not None:resumable_source_manager.save_state(AttioResumeConfig(offset=int(state["offset"])))resource = rest_api_resource(diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/canonical_descriptions.pyindex ef0e9e0f149d..4032b1e00ceb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/canonical_descriptions.py@@ -104,4 +104,18 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"created_at": "Time at which the member was added to the workspace.",},},+ "activities": {+ "description": "An activity type defined in the Attio workspace, such as a call or site visit. Lists the type definitions, not the individual activity records.",+ "docs_url": "https://docs.attio.com/rest-api/endpoint-reference/activities/list-activities",+ "columns": {+ "id": "Composite identifier for the activity type (workspace and activity id).",+ "activity_id": "Unique identifier for the activity type.",+ "api_slug": "URL-safe slug used to reference the activity type in the API.",+ "singular_noun": "Singular display name of the activity type.",+ "plural_noun": "Plural display name of the activity type.",+ "extends": "The standard schemas this activity type extends.",+ "is_system_activity": "Whether Attio defines the activity type, as opposed to a workspace user.",+ "created_at": "Time at which the activity type was created.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/settings.pyindex bedecc834251..c8bf308c5303 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/settings.py@@ -1,8 +1,9 @@-from dataclasses import dataclassfrom typing import Literal+from posthog.dataclasses import frozen-@dataclass++@frozenclass AttioEndpointConfig:name: strpath: str@@ -10,6 +11,7 @@ class AttioEndpointConfig:partition_key: str = "created_at"method: Literal["GET", "POST"] = "GET"page_size: int = 500+ pagination: Literal["offset", "cursor"] = "offset"# Attio API doesn't support updatedAt filtering, so only full refresh is supported@@ -69,6 +71,13 @@ ATTIO_ENDPOINTS: dict[str, AttioEndpointConfig] = {method="GET",primary_key="workspace_member_id",),+ "activities": AttioEndpointConfig(+ name="activities",+ path="/v2/activities",+ method="GET",+ primary_key="activity_id",+ pagination="cursor",+ ),}ENDPOINTS = tuple(ATTIO_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/source.pyindex ef10319b52f9..d15dc10d40b2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/source.py@@ -74,6 +74,7 @@ You can generate an API key in your Attio workspace settings. Check out [this gu- `note:read` - To read notes- `task:read` - To read tasks- `user_management:read` - To read workspace members+- `activity_configuration:read` - To read activity types""",iconPath="/static/services/attio.png",docsUrl="https://posthog.com/docs/cdp/sources/attio",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/tests/test_attio.py b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/tests/test_attio.pyindex 6ed2e29d99e2..685576b0661e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attio/tests/test_attio.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/attio/tests/test_attio.py@@ -88,3 +88,54 @@ class TestAttioResume:assert [request.json() for request in result.requests] == [{"sorts": [{"attribute": "created_at", "direction": "asc"}], "offset": 500, "limit": 500}]+++def _cursor_page(start: int, count: int, next_cursor: str | None) -> ScriptedResponse:+ rows = [+ {"id": {"workspace_id": "ws", "activity_id": f"row-{i}"}, "created_at": "2024-01-01"}+ for i in range(start, start + count)+ ]+ return ScriptedResponse(json={"data": rows, "pagination": {"next_cursor": next_cursor}})+++class TestAttioCursorPagination:+ @staticmethod+ def _driver() -> SourceDriver:+ return SourceDriver(AttioSource(), AttioSourceConfig(api_key="key"))++ @pytest.mark.parametrize(+ "resume_state, responses, expected_cursors, expected_rows, expected_states",+ [+ (+ None,+ [_cursor_page(0, 2, "c1"), _cursor_page(2, 2, "c2"), _cursor_page(4, 1, None)],+ [None, "c1", "c2"],+ 5,+ [AttioResumeConfig(cursor="c1"), AttioResumeConfig(cursor="c2")],+ ),+ (+ AttioResumeConfig(cursor="c2"),+ [_cursor_page(4, 2, "c3"), _cursor_page(6, 1, None)],+ ["c2", "c3"],+ 3,+ [AttioResumeConfig(cursor="c3")],+ ),+ ],+ )+ def test_activities_follow_next_cursor_and_stage_it(+ self,+ resume_state: AttioResumeConfig | None,+ responses: list[ScriptedResponse],+ expected_cursors: list[str | None],+ expected_rows: int,+ expected_states: list[AttioResumeConfig],+ ) -> None:+ result = self._driver().run("activities", responses, resume_state=resume_state)++ assert result.raised is None+ assert [request.param("cursor") for request in result.requests] == expected_cursors+ assert all(request.param("limit") == "500" for request in result.requests)+ assert all(request.param("offset") is None for request in result.requests)+ assert len(result.rows) == expected_rows+ assert "activity_id" in result.rows[0]+ assert result.saved_states == expected_states