Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 2 missing tables to the Lattice source (#120866)

Tom Owers43cbed0fc9e4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 50ffae1a8d83..da726d789ee4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -7152,9 +7152,9 @@ Diffed against: <https://developers.lattice.com/reference>
- [x] `GET /v1/reviewCycle/{id}/reviews (also /v1/reviewee/{id}/reviews)` — the actual performance reviews - the core analytical object of the review cycles already synced (high)
- [x] `GET /v1/reviewCycle/{id}/reviewees` — membership table mapping users to review cycles, needed for participation and completion analysis (high)
- [x] `GET /v1/goals/updates (and /v1/goals/{id}/updates)` — goal progress history - the state/transition trail behind the goals table; distinct from the synced 'updates' resource (high)
-- [ ] `GET /v1/user/{id}/tasks` — task records tied to goals and users, commonly wanted for follow-through analysis (medium)
+- [x] `GET /v1/user/{id}/tasks` — task records tied to goals and users, commonly wanted for follow-through analysis (medium)
- [x] `GET /v1/tags` — lookup resolving tag ids that appear on goals, users and feedback (medium)
-- [ ] `GET /v1/user/{id}/customAttributes (+ /v1/customAttribute/{id}, /v1/customAttributeValue/{id})` — custom HR attributes per user plus the lookup that decodes their ids - the main segmentation dimension (medium)
+- [x] `GET /v1/user/{id}/customAttributes (+ /v1/customAttribute/{id}, /v1/customAttributeValue/{id})` — custom HR attributes per user plus the lookup that decodes their ids - the main segmentation dimension (medium)
- [ ] `GET /v1/question/{id} and /v1/questionRevision/{id}` — lookup resolving question ids carried on reviews and feedback (fetch-by-id only, no list endpoint) (low)
- [ ] `GET /v1/competency/{id}` — competency lookup referenced by review questions and ratings (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/canonical_descriptions.py
index d27ca2076994..5b5c0564db01 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/canonical_descriptions.py
@@ -144,4 +144,23 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
goal="The goal the update relates to, if any.",
),
},
+ "user_tasks": {
+ "description": "A pending task assigned to a Lattice user, such as a review to write or a goal to update.",
+ "docs_url": "https://developers.lattice.com/reference/api_user_tasks",
+ "columns": {
+ "id": "Unique identifier for the task.",
+ "user_id": "ID of the user the task is assigned to.",
+ },
+ },
+ "user_custom_attributes": {
+ "description": "A custom HR attribute value set on a Lattice user, with the attribute definition it belongs to.",
+ "docs_url": "https://developers.lattice.com/reference/api_user_customattributes",
+ "columns": {
+ "user_id": "ID of the user the attribute value is set on.",
+ "custom_attribute_id": "ID of the custom attribute, copied from customAttribute.",
+ "custom_attribute_value_id": "ID of the custom attribute value, copied from customAttributeValue.",
+ "customAttribute": "The custom attribute definition: its name, type, and who can see it.",
+ "customAttributeValue": "The value of the custom attribute for this user, with its displayed text.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/lattice.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/lattice.py
index 6640e9c5bbdb..9ce8ab3bbba2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/lattice.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/lattice.py
@@ -1,5 +1,6 @@
import dataclasses
-from typing import Any, Optional
+from collections.abc import Callable
+from typing import Any, Optional, cast
from urllib.parse import urlencode
from requests import Request, Response
@@ -13,6 +14,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.res
build_dependent_resource,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
@@ -93,6 +95,23 @@ class LatticeCursorPaginator(BasePaginator):
self._has_next_page = True
+def _nested_id(value: Any) -> Optional[str]:
+ return value.get("id") if isinstance(value, dict) else None
+
+
+def _lift_custom_attribute_ids(row: dict[str, Any]) -> dict[str, Any]:
+ return {
+ **row,
+ "custom_attribute_id": _nested_id(row.get("customAttribute")),
+ "custom_attribute_value_id": _nested_id(row.get("customAttributeValue")),
+ }
+
+
+ROW_MAPS: dict[str, Callable[[dict[str, Any]], dict[str, Any]]] = {
+ "user_custom_attributes": _lift_custom_attribute_ids,
+}
+
+
def _base_url(region: str) -> str:
host = LATTICE_HOSTS.get(region)
if host is None:
@@ -166,6 +185,9 @@ def lattice_source(
resume_hook=save_fanout_checkpoint,
initial_paginator_state=resume.fanout_state if resume else None,
)
+ row_map = ROW_MAPS.get(endpoint)
+ if row_map is not None:
+ child = cast(Resource, child).add_map(row_map)
return SourceResponse(
name=endpoint,
items=lambda: child,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/settings.py
index bd7af5e8c8ec..4f32f3d7ad60 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/settings.py
@@ -17,6 +17,15 @@ REVIEW_CYCLE_FANOUT = DependentEndpointConfig(
parent_field_renames={"id": "review_cycle_id"},
)
+# Tasks and custom attributes are only listed per user, so they fan out over `/v1/users`.
+USER_FANOUT = DependentEndpointConfig(
+ parent_name="users",
+ resolve_param="user_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "user_id"},
+)
+
@dataclass(frozen=False)
class LatticeEndpointConfig:
@@ -80,6 +89,19 @@ LATTICE_ENDPOINTS: dict[str, LatticeEndpointConfig] = {
name="updates",
path="/v1/updates",
),
+ "user_tasks": LatticeEndpointConfig(
+ name="user_tasks",
+ path="/v1/user/{user_id}/tasks",
+ primary_keys=["user_id", "id"],
+ fanout=USER_FANOUT,
+ ),
+ "user_custom_attributes": LatticeEndpointConfig(
+ name="user_custom_attributes",
+ path="/v1/user/{user_id}/customAttributes",
+ # Rows only nest the attribute and value objects, so their ids are lifted into top-level key columns.
+ primary_keys=["user_id", "custom_attribute_id", "custom_attribute_value_id"],
+ fanout=USER_FANOUT,
+ ),
}
ENDPOINTS = tuple(LATTICE_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/tests/test_lattice.py b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/tests/test_lattice.py
index d09242762efd..5e6617684e8e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/tests/test_lattice.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/lattice/tests/test_lattice.py
@@ -156,3 +156,46 @@ class TestReviewCycleFanout:
assert [(row["review_cycle_id"], row["id"]) for row in result.rows] == [
(cycle, "r1") for cycle in expected_cycles
]
+
+
+class TestUserFanout:
+ @pytest.mark.parametrize(
+ "endpoint, child_row, expected_row_fields",
+ [
+ ("user_tasks", {"id": "t1"}, {"id": "t1"}),
+ (
+ "user_custom_attributes",
+ {"customAttribute": {"id": "ca1", "name": "Office"}, "customAttributeValue": {"id": "cv1"}},
+ {"custom_attribute_id": "ca1", "custom_attribute_value_id": "cv1"},
+ ),
+ # A row missing its nested objects must still sync rather than fail the run.
+ (
+ "user_custom_attributes",
+ {"customAttribute": None},
+ {"custom_attribute_id": None, "custom_attribute_value_id": None},
+ ),
+ ],
+ )
+ def test_fans_out_over_users(
+ self, endpoint: str, child_row: dict[str, Any], expected_row_fields: dict[str, Any]
+ ) -> None:
+ child_path = {"user_tasks": "tasks", "user_custom_attributes": "customAttributes"}[endpoint]
+ result = _driver().run(
+ endpoint,
+ [
+ _response([{"id": "u1"}, {"id": "u2"}]),
+ _response([child_row]),
+ _response([child_row]),
+ ],
+ )
+
+ assert result.raised is None
+ assert [url.split("?")[0] for url in result.urls] == [
+ "https://api.latticehq.com/v1/users",
+ f"https://api.latticehq.com/v1/user/u1/{child_path}",
+ f"https://api.latticehq.com/v1/user/u2/{child_path}",
+ ]
+ # Each row carries its user so the composite primary key stays unique across users.
+ assert [row["user_id"] for row in result.rows] == ["u1", "u2"]
+ for row in result.rows:
+ assert {key: row[key] for key in expected_row_fields} == expected_row_fields