Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 5 missing tables to the Guru source (#124467)

Tom Owersd0cbbd843b38
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index 847b2db287d9..d6fbb0756bc8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -5663,10 +5663,10 @@ Diffed against: <https://developer.getguru.com/reference/authentication>
- [x] `group members (GET /v1/groups/{id}/members)` — membership table joining the groups and members we already sync (high)
- [x] `tag categories and tags (GET /v1/teams/{id}/tagcategories, /tagcategories/tags)` — lookup resolving the tag IDs carried on synced cards (high)
- [x] `folders (GET /v1/folders, GET /v1/folders/{id}/items)` — the board/folder hierarchy that organizes cards - lookup plus card-to-folder membership (high)
-- [ ] `card comments (GET /v1/cards/{id}/comments)` — engagement and feedback events attached to synced cards (medium)
-- [ ] `card verifiers (GET /v1/cards/verifiers, /v1/cards/{id}/verifiers)` — verification ownership per card - drives knowledge-freshness reporting (medium)
-- [ ] `team stats (GET /v1/teams/{id}/stats)` — rolled-up workspace counters for trend dashboards (medium)
-- [ ] `knowledge alerts (GET /v1/alerts/delegated, /v1/alerts/{id}/stats/summary, /stats/users)` — alert delivery and per-user read state - announcement reach analysis (medium)
+- [x] `card comments (GET /v1/cards/{id}/comments)` — engagement and feedback events attached to synced cards (medium)
+- [x] `card verifiers (GET /v1/cards/verifiers, /v1/cards/{id}/verifiers)` — verification ownership per card - drives knowledge-freshness reporting (medium)
+- [x] `team stats (GET /v1/teams/{id}/stats)` — rolled-up workspace counters for trend dashboards (medium)
+- [x] `knowledge alerts (GET /v1/alerts/delegated, /v1/alerts/{id}/stats/summary, /stats/users)` — alert delivery and per-user read state - announcement reach analysis (medium)
- [ ] `templates (GET /v1/templates)` — lookup for the template a card was created from (medium)
- [ ] `collection access (GET /v1/collections/{id}/access, GET /v1/groups/{id}/collections)` — group-to-collection permission mapping, a membership-style join across two synced tables (low)
- [ ] `people (GET /v1/people/{id}, /v1/people/{id}/directreports)` — org chart attributes enriching members (low)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/canonical_descriptions.py
index cb14660112a6..0dcdb1e70e54 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/canonical_descriptions.py
@@ -141,4 +141,73 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"properties": "Event-specific properties. The most common is cardId, the card the event applies to.",
},
},
+ "card_comments": {
+ "description": "A comment on a card, with its replies and reactions.",
+ "docs_url": "https://developer.getguru.com/reference/getv1commentsgetcomments",
+ "columns": {
+ "id": "Unique identifier for the comment.",
+ "card": "The card the comment is on: its ID, title (preferredPhrase) and slug.",
+ "content": "Text of the comment.",
+ "status": "Whether the comment is OPEN or RESOLVED.",
+ "owner": "The user who wrote the comment.",
+ "totalReplies": "Number of replies to the comment.",
+ "replies": "Replies to the comment.",
+ "reactions": "Emoji reactions on the comment, with a count for each.",
+ "deleted": "Whether the comment is deleted.",
+ "dateCreated": "Time at which the comment was created.",
+ "lastModified": "Time at which the comment was last modified.",
+ },
+ },
+ "card_verifiers": {
+ "description": "A user or user group that must verify a card, one row per card and verifier.",
+ "docs_url": "https://developer.getguru.com/reference/getv1cardsverifiersgetverifiers",
+ "columns": {
+ "id": "ID of the verifier.",
+ "cardId": "ID of the card the verifier is assigned to.",
+ "type": "Whether the verifier is a user or a user-group.",
+ "user": "The verifying user, when type is user.",
+ "userGroup": "The verifying user group, when type is user-group.",
+ "dateCreated": "Time at which the verifier was assigned.",
+ },
+ },
+ "team_stats": {
+ "description": "A current counter for the team, such as its card count or its trust score.",
+ "docs_url": "https://developer.getguru.com/reference/getv1teamsgetteamstats",
+ "columns": {
+ "statKey": "Key of the stat in the team stats response.",
+ "type": "Type of the stat (team-card-count, team-trust-score, card-count, or collection-trust-score).",
+ "count": "Number of cards, for card-count stats.",
+ "trustedCount": "Number of trusted cards, for trust-score stats.",
+ "needsVerificationCount": "Number of cards that need verification, for trust-score stats.",
+ },
+ },
+ "knowledge_alerts": {
+ "description": "A sent knowledge alert (announcement) that asks a set of groups to read a card.",
+ "docs_url": "https://developer.getguru.com/reference/getv1announcementsgetdelegatedannouncements",
+ "columns": {
+ "alertId": "Unique identifier for the alert.",
+ "card": "The card the alert asks people to read.",
+ "cardTitle": "Title of the card.",
+ "groups": "Groups the alert was sent to.",
+ "note": "Note sent with the alert.",
+ "sentBy": "The user who sent the alert.",
+ "dateSent": "Time at which the alert was sent.",
+ "readCount": "Number of recipients who have read the alert.",
+ "unreadCount": "Number of recipients who have not read the alert.",
+ "percentRead": "Percentage of recipients who have read the alert.",
+ },
+ },
+ "knowledge_alert_users": {
+ "description": "A recipient of a knowledge alert, with when they viewed and read it.",
+ "docs_url": "https://developer.getguru.com/reference/getv1announcementsgetannouncementstatusers",
+ "columns": {
+ "alert_id": "ID of the knowledge alert.",
+ "email": "Email address of the recipient.",
+ "id": "Unique identifier for the recipient user.",
+ "firstName": "First name of the recipient.",
+ "lastName": "Last name of the recipient.",
+ "announcementViewedAt": "Time at which the recipient viewed the alert. Null if they have not.",
+ "announcementReadAt": "Time at which the recipient marked the alert as read. Null if they have not.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/guru.py b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/guru.py
index 1e22761b511e..c36f24b3f59c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/guru.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/guru.py
@@ -122,6 +122,21 @@ def _flatten_tag_category(category: dict[str, Any]) -> list[dict[str, Any]]:
]
+def _flatten_card_verifiers(card: dict[str, Any]) -> list[dict[str, Any]]:
+ # A verifier is a user or a user group. Fall back to that entity's id when the verifier omits
+ # its own, and use direct access so a verifier with neither fails loudly instead of a null key.
+ rows = []
+ for verifier in card.get("verifiers") or []:
+ verifier_id = verifier.get("id") or (verifier.get("user") or verifier.get("userGroup") or {})["id"]
+ rows.append({"cardId": card["id"], **verifier, "id": verifier_id})
+ return rows
+
+
+def _flatten_team_stats(body: dict[str, Any]) -> list[dict[str, Any]]:
+ # Team stats come back as one object that maps a stat key to its counters.
+ return [{"statKey": key, **stat} for key, stat in body["stats"].items()]
+
+
def _client_config(username: str, api_token: str) -> ClientConfig:
return {
"base_url": GURU_BASE_URL,
@@ -221,7 +236,7 @@ def guru_source(
"params": params,
# Guru returns a bare JSON array; a non-list 200 body means the response shape
# changed — fail loud instead of wrapping the stray object as a single row.
- "data_selector_required": True,
+ "data_selector_required": not config.single_object,
}
resource_config: EndpointResource = {
"name": endpoint,
@@ -233,6 +248,10 @@ def guru_source(
resource_config["data_map"] = _flatten_tag_category
elif endpoint == "analytics_events":
resource_config["data_map"] = _ensure_event_id
+ elif endpoint == "card_verifiers":
+ resource_config["data_map"] = _flatten_card_verifiers
+ elif endpoint == "team_stats":
+ resource_config["data_map"] = _flatten_team_stats
rest_config: RESTAPIConfig = {
"client": _client_config(username, api_token),
@@ -271,8 +290,8 @@ def guru_source(
partition_mode="datetime" if config.partition_key else None,
partition_format="month" if config.partition_key else None,
partition_keys=[config.partition_key] if config.partition_key else None,
- # Guru documents no order for the analytics export, so "desc" defers the watermark
- # commit to the end of the run instead of checkpointing it after each batch.
+ # Guru documents no order for the analytics export or the comment list, so "desc" defers
+ # the watermark commit to the end of the run instead of checkpointing it after each batch.
sort_mode="desc" if config.date_filter_param else "asc",
column_hints=resource.column_hints,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/settings.py
index 7f51827ba785..45255501788a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/settings.py
@@ -21,6 +21,8 @@ class GuruEndpointConfig:
# Query param that takes the incremental cursor as an inclusive lower bound. When unset,
# incremental endpoints filter with a Guru Query Language `q` expression instead.
date_filter_param: Optional[str] = None
+ # The endpoint returns one JSON object, not an array; its data map turns it into rows.
+ single_object: bool = False
fanout: Optional[DependentEndpointConfig] = None
@property
@@ -33,10 +35,13 @@ class GuruEndpointConfig:
return 0
+# Only sent alerts: scheduled alerts have no send date or read state yet.
+KNOWLEDGE_ALERT_PARAMS = {"view": "ALL_SENT", "sortField": "dateSent", "sortOrder": "ASC"}
+
# Guru responses are bare JSON arrays paginated via a `Link: <url>; rel="next-page"`
# header with an opaque continuation token. Only the card search surface (Guru Query
-# Language `lastModified >= <ISO8601>`) and the analytics event export (`fromDate`) support
-# a server-side date filter; the dimension tables are full refresh per run. A `{team_id}`
+# Language `lastModified >= <ISO8601>`), the analytics event export (`fromDate`) and the
+# comment list (`createdAfter`) support a server-side date filter; the dimension tables are full refresh per run. A `{team_id}`
# path placeholder is filled from the authenticated user's team.
GURU_ENDPOINTS: dict[str, GuruEndpointConfig] = {
"cards": GuruEndpointConfig(
@@ -122,6 +127,55 @@ GURU_ENDPOINTS: dict[str, GuruEndpointConfig] = {
},
],
),
+ "card_comments": GuruEndpointConfig(
+ name="card_comments",
+ # The team-wide comment list carries a `card` reference on each comment, so no per-card fan-out.
+ path="/comments",
+ partition_key="dateCreated",
+ date_filter_param="createdAfter",
+ incremental_fields=[
+ {
+ "label": "dateCreated",
+ "type": IncrementalFieldType.DateTime,
+ "field": "dateCreated",
+ "field_type": IncrementalFieldType.DateTime,
+ },
+ ],
+ ),
+ "card_verifiers": GuruEndpointConfig(
+ name="card_verifiers",
+ # Card search results embed each card's verifiers, so they are flattened from the card
+ # pages instead of one /cards/{id}/verifiers request per card.
+ path="/search/query",
+ # A user or group can verify several cards, so the verifier id is only unique per card.
+ primary_key=["cardId", "id"],
+ extra_params={"queryType": "cards", "maxResults": "50", "sortField": "dateCreated", "sortOrder": "asc"},
+ ),
+ "team_stats": GuruEndpointConfig(
+ name="team_stats",
+ path="/teams/{team_id}/stats",
+ primary_key="statKey",
+ single_object=True,
+ ),
+ "knowledge_alerts": GuruEndpointConfig(
+ name="knowledge_alerts",
+ path="/announcements/delegated",
+ primary_key="alertId",
+ extra_params=KNOWLEDGE_ALERT_PARAMS,
+ ),
+ "knowledge_alert_users": GuruEndpointConfig(
+ name="knowledge_alert_users",
+ path="/announcements/{announcementId}/stats/users",
+ primary_key=["alert_id", "email"],
+ fanout=DependentEndpointConfig(
+ parent_name="knowledge_alerts",
+ resolve_param="announcementId",
+ resolve_field="alertId",
+ include_from_parent=["alertId"],
+ parent_field_renames={"alertId": "alert_id"},
+ parent_params=KNOWLEDGE_ALERT_PARAMS,
+ ),
+ ),
}
ENDPOINTS = tuple(GURU_ENDPOINTS.keys())
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/tests/test_guru.py b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/tests/test_guru.py
index 86d60e2f9220..e2dbbf268ea6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/guru/tests/test_guru.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/guru/tests/test_guru.py
@@ -174,22 +174,54 @@ class TestTeamScopedEndpoints:
assert result.paths[1] == "/api/v1/teams/team-1/tagcategories"
assert result.rows == [{"categoryId": "cat-1", "categoryName": "Fruits", "id": "t1", "value": "Apple"}]
- def test_analytics_incremental_request_filters_by_from_date(self) -> None:
+ @pytest.mark.parametrize(
+ "endpoint, cursor_field, filter_param, path, needs_team",
+ [
+ ("analytics_events", "eventDate", "fromDate", "/api/v1/teams/team-1/analytics", True),
+ ("card_comments", "dateCreated", "createdAfter", "/api/v1/comments", False),
+ ],
+ )
+ @pytest.mark.parametrize("incremental", [True, False])
+ def test_date_filtered_request_shaping(
+ self, endpoint: str, cursor_field: str, filter_param: str, path: str, needs_team: bool, incremental: bool
+ ) -> None:
+ script = [_page([{"id": "e1", cursor_field: "2024-01-02T00:00:00.000+0000"}])]
result = _driver().run(
- "analytics_events",
- [_whoami(), _page([{"id": "e1", "eventDate": "2024-01-02T00:00:00.000+0000"}])],
- db_incremental_field_last_value=datetime(2024, 1, 1, tzinfo=UTC),
- incremental_field="eventDate",
+ endpoint,
+ [_whoami(), *script] if needs_team else script,
+ db_incremental_field_last_value=datetime(2024, 1, 1, tzinfo=UTC) if incremental else None,
+ incremental_field=cursor_field if incremental else None,
)
assert result.raised is None
- assert result.paths[1] == "/api/v1/teams/team-1/analytics"
- assert result.queries[1] == {"fromDate": ("2024-01-01T00:00:00.000+00:00",)}
+ assert result.paths[-1] == path
+ assert result.queries[-1] == ({filter_param: ("2024-01-01T00:00:00.000+00:00",)} if incremental else {})
assert [row["id"] for row in result.rows] == ["e1"]
- # Guru documents no order for the export, so the watermark must not checkpoint mid-run.
+ # Guru documents no order for these lists, so the watermark must not checkpoint mid-run.
assert result.response is not None
assert result.response.sort_mode == "desc"
+ def test_team_stats_object_is_split_into_one_row_per_stat(self) -> None:
+ body = {
+ "stats": {
+ "team-card-count": {"type": "team-card-count", "count": 120},
+ "team-trust-score": {"type": "team-trust-score", "trustedCount": 90, "needsVerificationCount": 30},
+ }
+ }
+ result = _driver().run("team_stats", [_whoami(), _page(body)])
+
+ assert result.raised is None
+ assert result.paths[1] == "/api/v1/teams/team-1/stats"
+ assert result.rows == [
+ {"statKey": "team-card-count", "type": "team-card-count", "count": 120},
+ {
+ "statKey": "team-trust-score",
+ "type": "team-trust-score",
+ "trustedCount": 90,
+ "needsVerificationCount": 30,
+ },
+ ]
+
def test_whoami_auth_failure_surfaces_as_http_error(self) -> None:
result = _driver().run("analytics_events", [ScriptedResponse(status=401)])
@@ -197,35 +229,75 @@ class TestTeamScopedEndpoints:
assert "401 Client Error: Unauthorized for url: https://api.getguru.com" in str(result.raised)
+class TestCardVerifiers:
+ def test_verifiers_are_flattened_per_card(self) -> None:
+ cards = [
+ {
+ "id": "c1",
+ "verifiers": [
+ {"id": "v1", "type": "user", "user": {"id": "u1", "email": "[email protected]"}},
+ {"type": "user-group", "userGroup": {"id": "grp1", "name": "Support"}},
+ ],
+ },
+ {"id": "c2", "verifiers": [{"id": "v1", "type": "user", "user": {"id": "u1"}}]},
+ {"id": "c3", "verifiers": []},
+ {"id": "c4"},
+ ]
+ result = _driver().run("card_verifiers", [_page(cards)])
+
+ assert result.raised is None
+ assert result.paths == ["/api/v1/search/query"]
+ assert result.requests[0].param("sortField") == "dateCreated"
+ # The same verifier on two cards stays two rows, and a group verifier without its own id
+ # takes the group id, so the (cardId, id) key is never null.
+ assert [(row["cardId"], row["id"], row["type"]) for row in result.rows] == [
+ ("c1", "v1", "user"),
+ ("c1", "grp1", "user-group"),
+ ("c2", "v1", "user"),
+ ]
+
+ def test_verifier_without_any_id_fails_loud(self) -> None:
+ result = _driver().run("card_verifiers", [_page([{"id": "c1", "verifiers": [{"type": "user"}]}])])
+
+ assert isinstance(result.raised, KeyError)
+
+
class TestFanoutEndpoints:
@pytest.mark.parametrize(
- "endpoint, parent_id, child_path, child_rows, expected",
+ "endpoint, parent_row, child_path, child_rows, expected",
[
(
"group_members",
- "g1",
+ {"id": "g1"},
"/groups/g1/members",
[{"id": "m1", "user": {"email": "[email protected]"}}],
[{"id": "m1", "user": {"email": "[email protected]"}, "group_id": "g1", "email": "[email protected]"}],
),
(
"folder_items",
- "f1",
+ {"id": "f1"},
"/folders/f1/items",
[{"id": "c1", "type": "card"}],
[{"id": "c1", "type": "card", "folder_id": "f1"}],
),
+ (
+ "knowledge_alert_users",
+ {"alertId": "a1", "cardTitle": "Holiday policy"},
+ "/announcements/a1/stats/users",
+ [{"id": "u1", "email": "[email protected]", "announcementReadAt": None}],
+ [{"id": "u1", "email": "[email protected]", "announcementReadAt": None, "alert_id": "a1"}],
+ ),
],
)
def test_child_rows_carry_parent_id(
self,
endpoint: str,
- parent_id: str,
+ parent_row: dict[str, Any],
child_path: str,
child_rows: list[dict[str, Any]],
expected: list[dict[str, Any]],
) -> None:
- result = _driver().run(endpoint, [_page([{"id": parent_id}]), _page(child_rows)])
+ result = _driver().run(endpoint, [_page([parent_row]), _page(child_rows)])
assert result.raised is None
assert result.paths[1].endswith(child_path)
@@ -235,6 +307,17 @@ class TestFanoutEndpoints:
assert all(row[key] is not None for row in result.rows for key in result.response.primary_keys)
+class TestKnowledgeAlerts:
+ @pytest.mark.parametrize("endpoint", ["knowledge_alerts", "knowledge_alert_users"])
+ def test_alert_list_requests_all_sent_alerts(self, endpoint: str) -> None:
+ # Without view=ALL_SENT Guru returns only the alerts the token owner sent.
+ result = _driver().run(endpoint, [_page([])])
+
+ assert result.raised is None
+ assert result.paths[0] == "/api/v1/announcements/delegated"
+ assert result.queries[0] == {"view": ("ALL_SENT",), "sortField": ("dateSent",), "sortOrder": ("ASC",)}
+
+
class TestGuruSourceResponse:
@pytest.mark.parametrize("endpoint", [name for name, config in GURU_ENDPOINTS.items() if config.fanout is None])
def test_response_metadata_per_endpoint(self, endpoint: str) -> None:
@@ -245,7 +328,9 @@ class TestGuruSourceResponse:
assert result.raised is None
assert result.response is not None
assert result.response.name == endpoint
- assert result.response.primary_keys == [config.primary_key]
+ assert result.response.primary_keys == (
+ config.primary_key if isinstance(config.primary_key, list) else [config.primary_key]
+ )
if config.partition_key:
assert result.response.partition_mode == "datetime"
assert result.response.partition_keys == [config.partition_key]