:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 2 missing tables to the Lattice source (#120866)
Tom Owers43cbed0fc9e4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 50ffae1a8d83..da726d789ee4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -7152,9 +7152,9 @@ Diffed against: <https://developers.lattice.com/reference>- [x] `GET /v1/reviewCycle/{id}/reviews (also /v1/reviewee/{id}/reviews)` — the actual performance reviews - the core analytical object of the review cycles already synced (high)- [x] `GET /v1/reviewCycle/{id}/reviewees` — membership table mapping users to review cycles, needed for participation and completion analysis (high)- [x] `GET /v1/goals/updates (and /v1/goals/{id}/updates)` — goal progress history - the state/transition trail behind the goals table; distinct from the synced 'updates' resource (high)-- [ ] `GET /v1/user/{id}/tasks` — task records tied to goals and users, commonly wanted for follow-through analysis (medium)+- [x] `GET /v1/user/{id}/tasks` — task records tied to goals and users, commonly wanted for follow-through analysis (medium)- [x] `GET /v1/tags` — lookup resolving tag ids that appear on goals, users and feedback (medium)-- [ ] `GET /v1/user/{id}/customAttributes (+ /v1/customAttribute/{id}, /v1/customAttributeValue/{id})` — custom HR attributes per user plus the lookup that decodes their ids - the main segmentation dimension (medium)+- [x] `GET /v1/user/{id}/customAttributes (+ /v1/customAttribute/{id}, /v1/customAttributeValue/{id})` — custom HR attributes per user plus the lookup that decodes their ids - the main segmentation dimension (medium)- [ ] `GET /v1/question/{id} and /v1/questionRevision/{id}` — lookup resolving question ids carried on reviews and feedback (fetch-by-id only, no list endpoint) (low)- [ ] `GET /v1/competency/{id}` — competency lookup referenced by review questions and ratings (low)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/canonical_descriptions.pyindex d27ca2076994..5b5c0564db01 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/canonical_descriptions.py@@ -144,4 +144,23 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {goal="The goal the update relates to, if any.",),},+ "user_tasks": {+ "description": "A pending task assigned to a Lattice user, such as a review to write or a goal to update.",+ "docs_url": "https://developers.lattice.com/reference/api_user_tasks",+ "columns": {+ "id": "Unique identifier for the task.",+ "user_id": "ID of the user the task is assigned to.",+ },+ },+ "user_custom_attributes": {+ "description": "A custom HR attribute value set on a Lattice user, with the attribute definition it belongs to.",+ "docs_url": "https://developers.lattice.com/reference/api_user_customattributes",+ "columns": {+ "user_id": "ID of the user the attribute value is set on.",+ "custom_attribute_id": "ID of the custom attribute, copied from customAttribute.",+ "custom_attribute_value_id": "ID of the custom attribute value, copied from customAttributeValue.",+ "customAttribute": "The custom attribute definition: its name, type, and who can see it.",+ "customAttributeValue": "The value of the custom attribute for this user, with its displayed text.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/lattice.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/lattice.pyindex 6640e9c5bbdb..9ce8ab3bbba2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/lattice.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/lattice.py@@ -1,5 +1,6 @@import dataclasses-from typing import Any, Optional+from collections.abc import Callable+from typing import Any, Optional, castfrom urllib.parse import urlencodefrom requests import Request, Response@@ -13,6 +14,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resbuild_dependent_resource,)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe@@ -93,6 +95,23 @@ class LatticeCursorPaginator(BasePaginator):self._has_next_page = True+def _nested_id(value: Any) -> Optional[str]:+ return value.get("id") if isinstance(value, dict) else None+++def _lift_custom_attribute_ids(row: dict[str, Any]) -> dict[str, Any]:+ return {+ **row,+ "custom_attribute_id": _nested_id(row.get("customAttribute")),+ "custom_attribute_value_id": _nested_id(row.get("customAttributeValue")),+ }+++ROW_MAPS: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = {+ "user_custom_attributes": _lift_custom_attribute_ids,+}++def _base_url(region: str) -> str:host = LATTICE_HOSTS.get(region)if host is None:@@ -166,6 +185,9 @@ def lattice_source(resume_hook=save_fanout_checkpoint,initial_paginator_state=resume.fanout_state if resume else None,)+ row_map = ROW_MAPS.get(endpoint)+ if row_map is not None:+ child = cast(Resource, child).add_map(row_map)return SourceResponse(name=endpoint,items=lambda: child,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/settings.pyindex bd7af5e8c8ec..4f32f3d7ad60 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/settings.py@@ -17,6 +17,15 @@ REVIEW_CYCLE_FANOUT = DependentEndpointConfig(parent_field_renames={"id": "review_cycle_id"},)+# Tasks and custom attributes are only listed per user, so they fan out over `/v1/users`.+USER_FANOUT = DependentEndpointConfig(+ parent_name="users",+ resolve_param="user_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "user_id"},+)+@dataclass(frozen=False)class LatticeEndpointConfig:@@ -80,6 +89,19 @@ LATTICE_ENDPOINTS: dict[str, LatticeEndpointConfig] = {name="updates",path="/v1/updates",),+ "user_tasks": LatticeEndpointConfig(+ name="user_tasks",+ path="/v1/user/{user_id}/tasks",+ primary_keys=["user_id", "id"],+ fanout=USER_FANOUT,+ ),+ "user_custom_attributes": LatticeEndpointConfig(+ name="user_custom_attributes",+ path="/v1/user/{user_id}/customAttributes",+ # Rows only nest the attribute and value objects, so their ids are lifted into top-level key columns.+ primary_keys=["user_id", "custom_attribute_id", "custom_attribute_value_id"],+ fanout=USER_FANOUT,+ ),}ENDPOINTS = tuple(LATTICE_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/tests/test_lattice.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/tests/test_lattice.pyindex d09242762efd..5e6617684e8e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/tests/test_lattice.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/tests/test_lattice.py@@ -156,3 +156,46 @@ class TestReviewCycleFanout:assert [(row["review_cycle_id"], row["id"]) for row in result.rows] == [(cycle, "r1") for cycle in expected_cycles]+++class TestUserFanout:+ @pytest.mark.parametrize(+ "endpoint, child_row, expected_row_fields",+ [+ ("user_tasks", {"id": "t1"}, {"id": "t1"}),+ (+ "user_custom_attributes",+ {"customAttribute": {"id": "ca1", "name": "Office"}, "customAttributeValue": {"id": "cv1"}},+ {"custom_attribute_id": "ca1", "custom_attribute_value_id": "cv1"},+ ),+ # A row missing its nested objects must still sync rather than fail the run.+ (+ "user_custom_attributes",+ {"customAttribute": None},+ {"custom_attribute_id": None, "custom_attribute_value_id": None},+ ),+ ],+ )+ def test_fans_out_over_users(+ self, endpoint: str, child_row: dict[str, Any], expected_row_fields: dict[str, Any]+ ) -> None:+ child_path = {"user_tasks": "tasks", "user_custom_attributes": "customAttributes"}[endpoint]+ result = _driver().run(+ endpoint,+ [+ _response([{"id": "u1"}, {"id": "u2"}]),+ _response([child_row]),+ _response([child_row]),+ ],+ )++ assert result.raised is None+ assert [url.split("?")[0] for url in result.urls] == [+ "https://api.latticehq.com/v1/users",+ f"https://api.latticehq.com/v1/user/u1/{child_path}",+ f"https://api.latticehq.com/v1/user/u2/{child_path}",+ ]+ # Each row carries its user so the composite primary key stays unique across users.+ assert [row["user_id"] for row in result.rows] == ["u1", "u2"]+ for row in result.rows:+ assert {key: row[key] for key in expected_row_fields} == expected_row_fields