:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 5 missing tables to the Guru source (#124467)
Tom Owersd0cbbd843b38
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 847b2db287d9..d6fbb0756bc8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -5663,10 +5663,10 @@ Diffed against: <https://developer.getguru.com/reference/authentication>- [x] `group members (GET /v1/groups/{id}/members)` — membership table joining the groups and members we already sync (high)- [x] `tag categories and tags (GET /v1/teams/{id}/tagcategories, /tagcategories/tags)` — lookup resolving the tag IDs carried on synced cards (high)- [x] `folders (GET /v1/folders, GET /v1/folders/{id}/items)` — the board/folder hierarchy that organizes cards - lookup plus card-to-folder membership (high)-- [ ] `card comments (GET /v1/cards/{id}/comments)` — engagement and feedback events attached to synced cards (medium)-- [ ] `card verifiers (GET /v1/cards/verifiers, /v1/cards/{id}/verifiers)` — verification ownership per card - drives knowledge-freshness reporting (medium)-- [ ] `team stats (GET /v1/teams/{id}/stats)` — rolled-up workspace counters for trend dashboards (medium)-- [ ] `knowledge alerts (GET /v1/alerts/delegated, /v1/alerts/{id}/stats/summary, /stats/users)` — alert delivery and per-user read state - announcement reach analysis (medium)+- [x] `card comments (GET /v1/cards/{id}/comments)` — engagement and feedback events attached to synced cards (medium)+- [x] `card verifiers (GET /v1/cards/verifiers, /v1/cards/{id}/verifiers)` — verification ownership per card - drives knowledge-freshness reporting (medium)+- [x] `team stats (GET /v1/teams/{id}/stats)` — rolled-up workspace counters for trend dashboards (medium)+- [x] `knowledge alerts (GET /v1/alerts/delegated, /v1/alerts/{id}/stats/summary, /stats/users)` — alert delivery and per-user read state - announcement reach analysis (medium)- [ ] `templates (GET /v1/templates)` — lookup for the template a card was created from (medium)- [ ] `collection access (GET /v1/collections/{id}/access, GET /v1/groups/{id}/collections)` — group-to-collection permission mapping, a membership-style join across two synced tables (low)- [ ] `people (GET /v1/people/{id}, /v1/people/{id}/directreports)` — org chart attributes enriching members (low)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/canonical_descriptions.pyindex cb14660112a6..0dcdb1e70e54 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/canonical_descriptions.py@@ -141,4 +141,73 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"properties": "Event-specific properties. The most common is cardId, the card the event applies to.",},},+ "card_comments": {+ "description": "A comment on a card, with its replies and reactions.",+ "docs_url": "https://developer.getguru.com/reference/getv1commentsgetcomments",+ "columns": {+ "id": "Unique identifier for the comment.",+ "card": "The card the comment is on: its ID, title (preferredPhrase) and slug.",+ "content": "Text of the comment.",+ "status": "Whether the comment is OPEN or RESOLVED.",+ "owner": "The user who wrote the comment.",+ "totalReplies": "Number of replies to the comment.",+ "replies": "Replies to the comment.",+ "reactions": "Emoji reactions on the comment, with a count for each.",+ "deleted": "Whether the comment is deleted.",+ "dateCreated": "Time at which the comment was created.",+ "lastModified": "Time at which the comment was last modified.",+ },+ },+ "card_verifiers": {+ "description": "A user or user group that must verify a card, one row per card and verifier.",+ "docs_url": "https://developer.getguru.com/reference/getv1cardsverifiersgetverifiers",+ "columns": {+ "id": "ID of the verifier.",+ "cardId": "ID of the card the verifier is assigned to.",+ "type": "Whether the verifier is a user or a user-group.",+ "user": "The verifying user, when type is user.",+ "userGroup": "The verifying user group, when type is user-group.",+ "dateCreated": "Time at which the verifier was assigned.",+ },+ },+ "team_stats": {+ "description": "A current counter for the team, such as its card count or its trust score.",+ "docs_url": "https://developer.getguru.com/reference/getv1teamsgetteamstats",+ "columns": {+ "statKey": "Key of the stat in the team stats response.",+ "type": "Type of the stat (team-card-count, team-trust-score, card-count, or collection-trust-score).",+ "count": "Number of cards, for card-count stats.",+ "trustedCount": "Number of trusted cards, for trust-score stats.",+ "needsVerificationCount": "Number of cards that need verification, for trust-score stats.",+ },+ },+ "knowledge_alerts": {+ "description": "A sent knowledge alert (announcement) that asks a set of groups to read a card.",+ "docs_url": "https://developer.getguru.com/reference/getv1announcementsgetdelegatedannouncements",+ "columns": {+ "alertId": "Unique identifier for the alert.",+ "card": "The card the alert asks people to read.",+ "cardTitle": "Title of the card.",+ "groups": "Groups the alert was sent to.",+ "note": "Note sent with the alert.",+ "sentBy": "The user who sent the alert.",+ "dateSent": "Time at which the alert was sent.",+ "readCount": "Number of recipients who have read the alert.",+ "unreadCount": "Number of recipients who have not read the alert.",+ "percentRead": "Percentage of recipients who have read the alert.",+ },+ },+ "knowledge_alert_users": {+ "description": "A recipient of a knowledge alert, with when they viewed and read it.",+ "docs_url": "https://developer.getguru.com/reference/getv1announcementsgetannouncementstatusers",+ "columns": {+ "alert_id": "ID of the knowledge alert.",+ "email": "Email address of the recipient.",+ "id": "Unique identifier for the recipient user.",+ "firstName": "First name of the recipient.",+ "lastName": "Last name of the recipient.",+ "announcementViewedAt": "Time at which the recipient viewed the alert. Null if they have not.",+ "announcementReadAt": "Time at which the recipient marked the alert as read. Null if they have not.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/guru.py b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/guru.pyindex 1e22761b511e..c36f24b3f59c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/guru.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/guru.py@@ -122,6 +122,21 @@ def _flatten_tag_category(category: dict[str, Any]) -> list[dict[str, Any]]:]+def _flatten_card_verifiers(card: dict[str, Any]) -> list[dict[str, Any]]:+ # A verifier is a user or a user group. Fall back to that entity's id when the verifier omits+ # its own, and use direct access so a verifier with neither fails loudly instead of a null key.+ rows = []+ for verifier in card.get("verifiers") or []:+ verifier_id = verifier.get("id") or (verifier.get("user") or verifier.get("userGroup") or {})["id"]+ rows.append({"cardId": card["id"], **verifier, "id": verifier_id})+ return rows+++def _flatten_team_stats(body: dict[str, Any]) -> list[dict[str, Any]]:+ # Team stats come back as one object that maps a stat key to its counters.+ return [{"statKey": key, **stat} for key, stat in body["stats"].items()]++def _client_config(username: str, api_token: str) -> ClientConfig:return {"base_url": GURU_BASE_URL,@@ -221,7 +236,7 @@ def guru_source("params": params,# Guru returns a bare JSON array; a non-list 200 body means the response shape# changed — fail loud instead of wrapping the stray object as a single row.- "data_selector_required": True,+ "data_selector_required": not config.single_object,}resource_config: EndpointResource = {"name": endpoint,@@ -233,6 +248,10 @@ def guru_source(resource_config["data_map"] = _flatten_tag_categoryelif endpoint == "analytics_events":resource_config["data_map"] = _ensure_event_id+ elif endpoint == "card_verifiers":+ resource_config["data_map"] = _flatten_card_verifiers+ elif endpoint == "team_stats":+ resource_config["data_map"] = _flatten_team_statsrest_config: RESTAPIConfig = {"client": _client_config(username, api_token),@@ -271,8 +290,8 @@ def guru_source(partition_mode="datetime" if config.partition_key else None,partition_format="month" if config.partition_key else None,partition_keys=[config.partition_key] if config.partition_key else None,- # Guru documents no order for the analytics export, so "desc" defers the watermark- # commit to the end of the run instead of checkpointing it after each batch.+ # Guru documents no order for the analytics export or the comment list, so "desc" defers+ # the watermark commit to the end of the run instead of checkpointing it after each batch.sort_mode="desc" if config.date_filter_param else "asc",column_hints=resource.column_hints,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/settings.pyindex 7f51827ba785..45255501788a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/settings.py@@ -21,6 +21,8 @@ class GuruEndpointConfig:# Query param that takes the incremental cursor as an inclusive lower bound. When unset,# incremental endpoints filter with a Guru Query Language `q` expression instead.date_filter_param: Optional[str] = None+ # The endpoint returns one JSON object, not an array; its data map turns it into rows.+ single_object: bool = Falsefanout: Optional[DependentEndpointConfig] = None@property@@ -33,10 +35,13 @@ class GuruEndpointConfig:return 0+# Only sent alerts: scheduled alerts have no send date or read state yet.+KNOWLEDGE_ALERT_PARAMS = {"view": "ALL_SENT", "sortField": "dateSent", "sortOrder": "ASC"}+# Guru responses are bare JSON arrays paginated via a `Link: <url>; rel="next-page"`# header with an opaque continuation token. Only the card search surface (Guru Query-# Language `lastModified >= <ISO8601>`) and the analytics event export (`fromDate`) support-# a server-side date filter; the dimension tables are full refresh per run. A `{team_id}`+# Language `lastModified >= <ISO8601>`), the analytics event export (`fromDate`) and the+# comment list (`createdAfter`) support a server-side date filter; the dimension tables are full refresh per run. A `{team_id}`# path placeholder is filled from the authenticated user's team.GURU_ENDPOINTS: dict[str, GuruEndpointConfig] = {"cards": GuruEndpointConfig(@@ -122,6 +127,55 @@ GURU_ENDPOINTS: dict[str, GuruEndpointConfig] = {},],),+ "card_comments": GuruEndpointConfig(+ name="card_comments",+ # The team-wide comment list carries a `card` reference on each comment, so no per-card fan-out.+ path="/comments",+ partition_key="dateCreated",+ date_filter_param="createdAfter",+ incremental_fields=[+ {+ "label": "dateCreated",+ "type": IncrementalFieldType.DateTime,+ "field": "dateCreated",+ "field_type": IncrementalFieldType.DateTime,+ },+ ],+ ),+ "card_verifiers": GuruEndpointConfig(+ name="card_verifiers",+ # Card search results embed each card's verifiers, so they are flattened from the card+ # pages instead of one /cards/{id}/verifiers request per card.+ path="/search/query",+ # A user or group can verify several cards, so the verifier id is only unique per card.+ primary_key=["cardId", "id"],+ extra_params={"queryType": "cards", "maxResults": "50", "sortField": "dateCreated", "sortOrder": "asc"},+ ),+ "team_stats": GuruEndpointConfig(+ name="team_stats",+ path="/teams/{team_id}/stats",+ primary_key="statKey",+ single_object=True,+ ),+ "knowledge_alerts": GuruEndpointConfig(+ name="knowledge_alerts",+ path="/announcements/delegated",+ primary_key="alertId",+ extra_params=KNOWLEDGE_ALERT_PARAMS,+ ),+ "knowledge_alert_users": GuruEndpointConfig(+ name="knowledge_alert_users",+ path="/announcements/{announcementId}/stats/users",+ primary_key=["alert_id", "email"],+ fanout=DependentEndpointConfig(+ parent_name="knowledge_alerts",+ resolve_param="announcementId",+ resolve_field="alertId",+ include_from_parent=["alertId"],+ parent_field_renames={"alertId": "alert_id"},+ parent_params=KNOWLEDGE_ALERT_PARAMS,+ ),+ ),}ENDPOINTS = tuple(GURU_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/tests/test_guru.py b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/tests/test_guru.pyindex 86d60e2f9220..e2dbbf268ea6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/tests/test_guru.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/tests/test_guru.py@@ -174,22 +174,54 @@ class TestTeamScopedEndpoints:assert result.paths[1] == "/api/v1/teams/team-1/tagcategories"assert result.rows == [{"categoryId": "cat-1", "categoryName": "Fruits", "id": "t1", "value": "Apple"}]- def test_analytics_incremental_request_filters_by_from_date(self) -> None:+ @pytest.mark.parametrize(+ "endpoint, cursor_field, filter_param, path, needs_team",+ [+ ("analytics_events", "eventDate", "fromDate", "/api/v1/teams/team-1/analytics", True),+ ("card_comments", "dateCreated", "createdAfter", "/api/v1/comments", False),+ ],+ )+ @pytest.mark.parametrize("incremental", [True, False])+ def test_date_filtered_request_shaping(+ self, endpoint: str, cursor_field: str, filter_param: str, path: str, needs_team: bool, incremental: bool+ ) -> None:+ script = [_page([{"id": "e1", cursor_field: "2024-01-02T00:00:00.000+0000"}])]result = _driver().run(- "analytics_events",- [_whoami(), _page([{"id": "e1", "eventDate": "2024-01-02T00:00:00.000+0000"}])],- db_incremental_field_last_value=datetime(2024, 1, 1, tzinfo=UTC),- incremental_field="eventDate",+ endpoint,+ [_whoami(), *script] if needs_team else script,+ db_incremental_field_last_value=datetime(2024, 1, 1, tzinfo=UTC) if incremental else None,+ incremental_field=cursor_field if incremental else None,)assert result.raised is None- assert result.paths[1] == "/api/v1/teams/team-1/analytics"- assert result.queries[1] == {"fromDate": ("2024-01-01T00:00:00.000+00:00",)}+ assert result.paths[-1] == path+ assert result.queries[-1] == ({filter_param: ("2024-01-01T00:00:00.000+00:00",)} if incremental else {})assert [row["id"] for row in result.rows] == ["e1"]- # Guru documents no order for the export, so the watermark must not checkpoint mid-run.+ # Guru documents no order for these lists, so the watermark must not checkpoint mid-run.assert result.response is not Noneassert result.response.sort_mode == "desc"+ def test_team_stats_object_is_split_into_one_row_per_stat(self) -> None:+ body = {+ "stats": {+ "team-card-count": {"type": "team-card-count", "count": 120},+ "team-trust-score": {"type": "team-trust-score", "trustedCount": 90, "needsVerificationCount": 30},+ }+ }+ result = _driver().run("team_stats", [_whoami(), _page(body)])++ assert result.raised is None+ assert result.paths[1] == "/api/v1/teams/team-1/stats"+ assert result.rows == [+ {"statKey": "team-card-count", "type": "team-card-count", "count": 120},+ {+ "statKey": "team-trust-score",+ "type": "team-trust-score",+ "trustedCount": 90,+ "needsVerificationCount": 30,+ },+ ]+def test_whoami_auth_failure_surfaces_as_http_error(self) -> None:result = _driver().run("analytics_events", [ScriptedResponse(status=401)])@@ -197,35 +229,75 @@ class TestTeamScopedEndpoints:assert "401 Client Error: Unauthorized for url: https://api.getguru.com" in str(result.raised)+class TestCardVerifiers:+ def test_verifiers_are_flattened_per_card(self) -> None:+ cards = [+ {+ "id": "c1",+ "verifiers": [+ {"id": "v1", "type": "user", "user": {"id": "u1", "email": "[email protected]"}},+ {"type": "user-group", "userGroup": {"id": "grp1", "name": "Support"}},+ ],+ },+ {"id": "c2", "verifiers": [{"id": "v1", "type": "user", "user": {"id": "u1"}}]},+ {"id": "c3", "verifiers": []},+ {"id": "c4"},+ ]+ result = _driver().run("card_verifiers", [_page(cards)])++ assert result.raised is None+ assert result.paths == ["/api/v1/search/query"]+ assert result.requests[0].param("sortField") == "dateCreated"+ # The same verifier on two cards stays two rows, and a group verifier without its own id+ # takes the group id, so the (cardId, id) key is never null.+ assert [(row["cardId"], row["id"], row["type"]) for row in result.rows] == [+ ("c1", "v1", "user"),+ ("c1", "grp1", "user-group"),+ ("c2", "v1", "user"),+ ]++ def test_verifier_without_any_id_fails_loud(self) -> None:+ result = _driver().run("card_verifiers", [_page([{"id": "c1", "verifiers": [{"type": "user"}]}])])++ assert isinstance(result.raised, KeyError)++class TestFanoutEndpoints:@pytest.mark.parametrize(- "endpoint, parent_id, child_path, child_rows, expected",+ "endpoint, parent_row, child_path, child_rows, expected",[("group_members",- "g1",+ {"id": "g1"},"/groups/g1/members",[{"id": "m1", "user": {"email": "[email protected]"}}],[{"id": "m1", "user": {"email": "[email protected]"}, "group_id": "g1", "email": "[email protected]"}],),("folder_items",- "f1",+ {"id": "f1"},"/folders/f1/items",[{"id": "c1", "type": "card"}],[{"id": "c1", "type": "card", "folder_id": "f1"}],),+ (+ "knowledge_alert_users",+ {"alertId": "a1", "cardTitle": "Holiday policy"},+ "/announcements/a1/stats/users",+ [{"id": "u1", "email": "[email protected]", "announcementReadAt": None}],+ [{"id": "u1", "email": "[email protected]", "announcementReadAt": None, "alert_id": "a1"}],+ ),],)def test_child_rows_carry_parent_id(self,endpoint: str,- parent_id: str,+ parent_row: dict[str, Any],child_path: str,child_rows: list[dict[str, Any]],expected: list[dict[str, Any]],) -> None:- result = _driver().run(endpoint, [_page([{"id": parent_id}]), _page(child_rows)])+ result = _driver().run(endpoint, [_page([parent_row]), _page(child_rows)])assert result.raised is Noneassert result.paths[1].endswith(child_path)@@ -235,6 +307,17 @@ class TestFanoutEndpoints:assert all(row[key] is not None for row in result.rows for key in result.response.primary_keys)+class TestKnowledgeAlerts:+ @pytest.mark.parametrize("endpoint", ["knowledge_alerts", "knowledge_alert_users"])+ def test_alert_list_requests_all_sent_alerts(self, endpoint: str) -> None:+ # Without view=ALL_SENT Guru returns only the alerts the token owner sent.+ result = _driver().run(endpoint, [_page([])])++ assert result.raised is None+ assert result.paths[0] == "/api/v1/announcements/delegated"+ assert result.queries[0] == {"view": ("ALL_SENT",), "sortField": ("dateSent",), "sortOrder": ("ASC",)}++class TestGuruSourceResponse:@pytest.mark.parametrize("endpoint", [name for name, config in GURU_ENDPOINTS.items() if config.fanout is None])def test_response_metadata_per_endpoint(self, endpoint: str) -> None:@@ -245,7 +328,9 @@ class TestGuruSourceResponse:assert result.raised is Noneassert result.response is not Noneassert result.response.name == endpoint- assert result.response.primary_keys == [config.primary_key]+ assert result.response.primary_keys == (+ config.primary_key if isinstance(config.primary_key, list) else [config.primary_key]+ )if config.partition_key:assert result.response.partition_mode == "datetime"assert result.response.partition_keys == [config.partition_key]