Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Vitally source (#128721)

Tom Owers86417da4512b
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index c8218af2f135..ca1e20c09630 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -13108,10 +13108,10 @@ Today (10): `Accounts`, `Conversations`, `Custom_Objects`, `Messages`, `NPS_Resp
Diffed against: <https://docs.vitally.io/en/collections/10410457-rest-api>
-- [ ] `GET /resources/admins` — lookup resolving the CSM/admin ids that appear on accounts, tasks, notes and conversations (high)
-- [ ] `GET /resources/projectTemplates` — lookup resolving the template behind each Project we already sync (high)
-- [ ] `GET /resources/meetings (and /resources/accounts/{id}/meetings)` — customer meeting activity records, a core engagement object with no equivalent today (high)
-- [ ] `GET /resources/projectCategories` — lookup resolving the category grouping on project templates and projects (medium)
+- [x] `GET /resources/admins` — lookup resolving the CSM/admin ids that appear on accounts, tasks, notes and conversations (high)
+- [x] `GET /resources/projectTemplates` — lookup resolving the template behind each Project we already sync (high)
+- [x] `GET /resources/meetings (and /resources/accounts/{id}/meetings)` — customer meeting activity records, a core engagement object with no equivalent today (high)
+- [x] `GET /resources/projectCategories` — lookup resolving the category grouping on project templates and projects (medium)
- [ ] `GET /resources/surveys/{surveyId}/responses` — custom survey responses — analytical feedback data distinct from the NPS responses already synced (medium)
- [ ] `GET /resources/customFields?model={model}` — custom trait definitions that give names and types to the opaque trait keys on accounts, users and custom objects (medium)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/canonical_descriptions.py
index bbf44aa445a5..30dea0a3ca54 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/canonical_descriptions.py
@@ -134,4 +134,59 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
conversation_updated_at="Time at which the parent conversation was last updated.",
),
},
+ "Admins": {
+ "description": "A Vitally admin (team member), such as the account owner or CSM referenced on other objects.",
+ "docs_url": "https://docs.vitally.io/en/articles/9880663-rest-api-admins",
+ "columns": {
+ "id": "Vitally's unique ID for the admin.",
+ "name": "The full name of the admin.",
+ "email": "The email of the admin.",
+ "licenseStatus": "The license status of the admin: 'active', 'deactivated' or 'auto-created'.",
+ "role": "The team member role assigned to the admin, for example 'leader'.",
+ },
+ },
+ "Meetings": {
+ "description": "A customer meeting recorded in Vitally, with its participants and linked accounts.",
+ "docs_url": "https://docs.vitally.io/en/articles/14501660-rest-api-meetings",
+ "columns": _columns(
+ title="The title of the meeting.",
+ description="The description or body of the meeting.",
+ location="The location of the meeting.",
+ startDate="Date-only start value of the meeting, when a time is not applicable.",
+ startDateTime="Time at which the meeting starts.",
+ startTimeZone="Time zone of the meeting start time.",
+ endDate="Date-only end value of the meeting, when a time is not applicable.",
+ endDateTime="Time at which the meeting ends.",
+ endTimeZone="Time zone of the meeting end time.",
+ externalStatus="Status of the meeting as reported by the external source, for example 'cancelled'.",
+ recordingUrl="URL of the meeting recording.",
+ summary="A summary of the meeting.",
+ keyPoints="Key points from the meeting.",
+ riskAssessment="A risk assessment associated with the meeting.",
+ source="Name of the integration that created the meeting, for example 'gong'.",
+ participants="Participants of the meeting: Vitally admins, customer users and external attendees.",
+ accounts="Vitally accounts associated with the meeting, resolved from its participants.",
+ organizations="Vitally organizations associated with the meeting, resolved from its participants.",
+ notes="Notes linked to the meeting.",
+ tasks="Tasks linked to the meeting.",
+ traits="Custom traits set on the meeting.",
+ archivedAt="Time at which the meeting was archived.",
+ ),
+ },
+ "Project_Templates": {
+ "description": "A template that Vitally uses to create projects.",
+ "docs_url": "https://docs.vitally.io/en/articles/9880869-rest-api-project-templates",
+ "columns": _columns(
+ name="The title of the project template.",
+ projectName="The name of the projects that the template creates.",
+ description="The description of the project template.",
+ projectCategoryId="ID of the project category of the template.",
+ traits="Custom project traits set on the template.",
+ ),
+ },
+ "Project_Categories": {
+ "description": "A category that groups Vitally project templates.",
+ "docs_url": "https://docs.vitally.io/en/articles/9880869-rest-api-project-templates",
+ "columns": _columns(name="The title of the project category."),
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/settings.py
index 32c5e918d0f5..6901e3df79cb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/settings.py
@@ -11,8 +11,15 @@ ENDPOINTS = (
"NPS_Responses",
"Custom_Objects",
"Messages",
+ "Admins",
+ "Meetings",
+ "Project_Templates",
+ "Project_Categories",
)
+# Rows of these endpoints carry no documented `createdAt`, so they skip the `created_at` partition.
+UNPARTITIONED_ENDPOINTS = ("Admins", "Meetings")
+
# Prefix used for schemas that sync the *instances* of a Vitally custom object.
# The `Custom_Objects` static endpoint syncs the definitions; one schema per object
# (named `Custom_Object_<machineName>`) syncs each object's actual records.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/source.py
index 9160273b608d..2d945dbccb79 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/source.py
@@ -30,6 +30,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.se
CUSTOM_OBJECT_SCHEMA_PREFIX,
ENDPOINTS as VITALLY_ENDPOINTS,
INCREMENTAL_FIELDS as VITALLY_INCREMENTAL_FIELDS,
+ UNPARTITIONED_ENDPOINTS,
UPDATED_AT_INCREMENTAL_FIELD,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.vitally.vitally import (
@@ -162,6 +163,7 @@ class VitallySource(ResumableSource[VitallySourceConfig, VitallyResumeConfig]):
resumable_source_manager: ResumableSourceManager[VitallyResumeConfig],
inputs: SourceInputs,
) -> SourceResponse:
+ is_partitioned = inputs.schema_name not in UNPARTITIONED_ENDPOINTS
return SourceResponse(
name=inputs.schema_name,
on_complete=resumable_source_manager.clear_state,
@@ -180,11 +182,11 @@ class VitallySource(ResumableSource[VitallySourceConfig, VitallyResumeConfig]):
resumable_source_manager=resumable_source_manager,
),
primary_keys=["id"],
- partition_count=1, # this enables partitioning
- partition_size=1, # this enables partitioning
- partition_mode="datetime",
- partition_format="week",
- partition_keys=["created_at"],
+ partition_count=1 if is_partitioned else None, # this enables partitioning
+ partition_size=1 if is_partitioned else None, # this enables partitioning
+ partition_mode="datetime" if is_partitioned else None,
+ partition_format="week" if is_partitioned else None,
+ partition_keys=["created_at"] if is_partitioned else None,
# `sortBy` carries no direction, so only the Messages read claims an order.
sort_mode="desc" if inputs.schema_name == "Messages" else None,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/tests/test_vitally.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/tests/test_vitally.py
index e3aa9d6ff3ca..63dff4aee8dd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/tests/test_vitally.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/tests/test_vitally.py
@@ -234,6 +234,55 @@ class TestResume:
]
+class TestFullRefreshEndpoints:
+ @pytest.mark.parametrize(
+ "endpoint, expected_path, expected_sort_by, expected_partition_keys",
+ [
+ pytest.param("Admins", "/resources/admins", None, None, id="admins"),
+ pytest.param("Meetings", "/resources/meetings", "createdAt", None, id="meetings"),
+ pytest.param(
+ "Project_Templates", "/resources/projectTemplates", "createdAt", ["created_at"], id="project_templates"
+ ),
+ pytest.param(
+ "Project_Categories",
+ "/resources/projectCategories",
+ "createdAt",
+ ["created_at"],
+ id="project_categories",
+ ),
+ ],
+ )
+ def test_walks_every_page(
+ self,
+ endpoint: str,
+ expected_path: str,
+ expected_sort_by: str | None,
+ expected_partition_keys: list[str] | None,
+ ) -> None:
+ driver = SourceDriver(
+ VitallySource(),
+ VitallySourceConfig(secret_token="secret", region=VitallyRegionConfig(subdomain="", selection="EU")),
+ )
+
+ result = driver.run(
+ endpoint,
+ [
+ ScriptedResponse(json={"results": [{"id": "row-1"}, {"id": "row-2"}], "next": "cursor-2"}),
+ ScriptedResponse(json={"results": [{"id": "row-3"}], "next": None}),
+ ],
+ )
+
+ assert result.raised is None
+ assert result.paths == [expected_path, expected_path]
+ assert result.params("from") == [None, "cursor-2"]
+ assert result.params("sortBy") == [expected_sort_by, expected_sort_by]
+ assert result.params("updatedAt") == [None, None]
+ assert [row["id"] for row in result.rows] == ["row-1", "row-2", "row-3"]
+ assert result.response is not None
+ assert result.response.partition_keys == expected_partition_keys
+ assert result.final_state == VitallyResumeConfig(completed=True)
+
+
class TestGetCustomObjectRecordsResource:
@pytest.mark.parametrize("should_use_incremental_field", [False, True])
def test_builds_resource(self, should_use_incremental_field):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/vitally.py b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/vitally.py
index f880b64611d1..d8e0ad3247e6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/vitally.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/vitally/vitally.py
@@ -280,6 +280,53 @@ def get_resource(name: str, should_use_incremental_field: bool) -> EndpointResou
},
"table_format": "delta",
},
+ # The lookup and meeting endpoints below are full refresh only: Vitally documents no server-side
+ # `updatedAt` filter for them. `sortBy=createdAt` keeps each row on one page while rows update mid-crawl.
+ "Admins": {
+ "name": "Admins",
+ "table_name": "admins",
+ "write_disposition": "replace",
+ "endpoint": {
+ "data_selector": "results",
+ "path": "/resources/admins",
+ # Admins have no `createdAt`, so they keep the default order.
+ "params": {"limit": 100},
+ },
+ "table_format": "delta",
+ },
+ "Meetings": {
+ "name": "Meetings",
+ "table_name": "meetings",
+ "write_disposition": "replace",
+ "endpoint": {
+ "data_selector": "results",
+ "path": "/resources/meetings",
+ "params": {"limit": 100, "sortBy": "createdAt"},
+ },
+ "table_format": "delta",
+ },
+ "Project_Templates": {
+ "name": "Project_Templates",
+ "table_name": "project_templates",
+ "write_disposition": "replace",
+ "endpoint": {
+ "data_selector": "results",
+ "path": "/resources/projectTemplates",
+ "params": {"limit": 100, "sortBy": "createdAt"},
+ },
+ "table_format": "delta",
+ },
+ "Project_Categories": {
+ "name": "Project_Categories",
+ "table_name": "project_categories",
+ "write_disposition": "replace",
+ "endpoint": {
+ "data_selector": "results",
+ "path": "/resources/projectCategories",
+ "params": {"limit": 100, "sortBy": "createdAt"},
+ },
+ "table_format": "delta",
+ },
}
return resources[name]