:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Bugherd source (#119941)
Tom Owers16a8123b6929
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex cff80bf9fc17..cd80937daaad 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -1937,9 +1937,9 @@ Diffed against: <https://docs.bugherd.com/api/openapi.yaml>- [x] `GET /api_v2/projects/{project_id}/columns.json` — lookup resolving the board column/status a task sits in (high)- [x] `GET /api_v2/projects/{project_id}/tasks/archive.json` — archived (closed-out) tasks live behind a separate endpoint, so completed work is entirely missing from the tasks table (high)- [x] `GET /api_v2/projects/{project_id}/tasks/feedback.json` — the unsorted feedback inbox — tasks not yet promoted to the board (medium)-- [ ] `GET /api_v2/projects/{project_id}/tasks/{task_id}/attachments.json` — screenshots and files attached to a task, joinable to tasks (medium)-- [ ] `GET /api_v2/users/{user_id}/projects.json` — membership join between users and projects; neither existing table carries the relationship (medium)-- [ ] `GET /api_v2/users/members.json and GET /api_v2/users/guests.json` — splits the flat users list into members vs guests, which is how BugHerd seats are licensed (medium)+- [x] `GET /api_v2/projects/{project_id}/tasks/{task_id}/attachments.json` — screenshots and files attached to a task, joinable to tasks (medium)+- [x] `GET /api_v2/users/{user_id}/projects.json` — membership join between users and projects; neither existing table carries the relationship (medium)+- [x] `GET /api_v2/users/members.json and GET /api_v2/users/guests.json` — splits the flat users list into members vs guests, which is how BugHerd seats are licensed (medium)- [ ] `GET /api_v2/projects/active.json` — distinguishes active from archived projects without inspecting each project (low)Note: docs.bugherd.com/api is a Scalar shell; the real spec is at /api/openapi.yaml (70KB, 27 paths). Note that the connector's Tasks stream hits /api_v2/projects/{id}/tasks.json, which is a separate endpoint from the archived and feedback task lists — archived tasks are therefore not synced at all, not merely filtered.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/canonical_descriptions.pyindex 5dbbfcef369e..9d6f90e0ac8e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/canonical_descriptions.py@@ -33,6 +33,14 @@ _TASK_COLUMNS = {"admin_link": "Task board URL for the task; login required.",}+# The members and guests lists return the same User object as the full users list.+_USER_COLUMNS = {+ "id": "Unique identifier for the user.",+ "email": "The user's email address.",+ "display_name": "The user's display name.",+ "avatar_url": "URL of the user's avatar image.",+}+CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"Organization": {"description": "Top-level details about your BugHerd organization.",@@ -45,12 +53,17 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"Users": {"description": "Team members and guests who belong to your BugHerd organization.","docs_url": _API_DOCS_URL,- "columns": {- "id": "Unique identifier for the user.",- "email": "The user's email address.",- "display_name": "The user's display name.",- "avatar_url": "URL of the user's avatar image.",- },+ "columns": _USER_COLUMNS,+ },+ "Members": {+ "description": "Team members (non-guest users) in your BugHerd organization.",+ "docs_url": _API_DOCS_URL,+ "columns": _USER_COLUMNS,+ },+ "Guests": {+ "description": "Guest users in your BugHerd organization: external collaborators invited to specific projects.",+ "docs_url": _API_DOCS_URL,+ "columns": _USER_COLUMNS,},"Projects": {"description": "Websites/projects tracked for visual feedback and bug reporting in BugHerd.",@@ -112,4 +125,27 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"is_private": "Whether the comment is visible only to team members, not guests/clients.",},},+ "TaskAttachments": {+ "description": "Screenshots and other files attached to a task.",+ "docs_url": _API_DOCS_URL,+ "columns": {+ "id": "Identifier for the attachment, unique within its task.",+ "task_id": "ID of the task the file is attached to.",+ "project_id": "ID of the project the task belongs to.",+ "file_name": "Name of the attached file.",+ "url": "URL of the attached file.",+ "created_at": "Timestamp when the file was attached.",+ },+ },+ "UserProjects": {+ "description": "Project memberships: one row for each project a user belongs to.",+ "docs_url": _API_DOCS_URL,+ "columns": {+ "user_id": "ID of the user who belongs to the project.",+ "id": "ID of the project the user belongs to.",+ "name": "The project's display name.",+ "created_at": "Timestamp when the project was created.",+ "is_active": "Whether the project is active.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/settings.pyindex 296a3517763c..ba861ee6b673 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/settings.py@@ -17,6 +17,14 @@ PROJECT_ID_FANOUT = DependentEndpointConfig(parent_field_renames={"id": "project_id"},)+USER_ID_FANOUT = DependentEndpointConfig(+ parent_name="Users",+ resolve_param="user_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "user_id"},+)+@dataclass(frozen=True)class BugherdChainedFanoutConfig:@@ -35,7 +43,7 @@ class BugherdChainedFanoutConfig:parent_field_renames: dict[str, str]-TASK_COMMENTS_FANOUT = BugherdChainedFanoutConfig(+TASK_ID_FANOUT = BugherdChainedFanoutConfig(parent_name="Tasks",resolve_fields={"project_id": "project_id", "task_id": "id"},include_from_parent=["id", "project_id"],@@ -82,6 +90,19 @@ BUGHERD_ENDPOINTS: dict[str, BugherdEndpointConfig] = {path="/api_v2/users.json",data_selector="users",),+ # The same User objects as `Users`, split by seat type: team members versus guests.+ # The User schema carries no member/guest flag, so these lists are the only place+ # the distinction appears.+ "Members": BugherdEndpointConfig(+ name="Members",+ path="/api_v2/users/members.json",+ data_selector="users",+ ),+ "Guests": BugherdEndpointConfig(+ name="Guests",+ path="/api_v2/users/guests.json",+ data_selector="users",+ ),"Projects": BugherdEndpointConfig(name="Projects",path="/api_v2/projects.json",@@ -155,7 +176,30 @@ BUGHERD_ENDPOINTS: dict[str, BugherdEndpointConfig] = {# so treat the ID as task-scoped rather than global.primary_key=["task_id", "id"],partition_key="created_at",- chained_fanout=TASK_COMMENTS_FANOUT,+ chained_fanout=TASK_ID_FANOUT,+ ),+ # Screenshots and files on a task. Like comments, the endpoint has no timestamp+ # filter, so full refresh.+ "TaskAttachments": BugherdEndpointConfig(+ name="TaskAttachments",+ path="/api_v2/projects/{project_id}/tasks/{task_id}/attachments.json",+ data_selector="attachments",+ # Addressed by project, task and attachment ID, so the ID is task-scoped.+ primary_key=["task_id", "id"],+ partition_key="created_at",+ chained_fanout=TASK_ID_FANOUT,+ ),+ # The user-to-project membership join: one row per project a user belongs to. Each+ # row is a full Project object, so `id` is the project ID. The `created_since`+ # filter only reports projects created after the watermark, which would miss a user+ # added to an older project, so this stays full refresh.+ "UserProjects": BugherdEndpointConfig(+ name="UserProjects",+ path="/api_v2/users/{user_id}/projects.json",+ data_selector="projects",+ primary_key=["user_id", "id"],+ partition_key="created_at",+ fanout=USER_ID_FANOUT,),}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/tests/test_bugherd.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/tests/test_bugherd.pyindex 75fc6fe79923..2bc0659856ac 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/tests/test_bugherd.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/tests/test_bugherd.py@@ -154,6 +154,33 @@ class TestBugherdFanout:assert result.params("created_since") == [None, "2024-01-01T00:00:00Z", "2024-01-01T00:00:00Z", None]assert result.params("updated_since") == [None, None, None, None]+ def test_user_projects_fans_out_over_users_and_stays_full_refresh(self) -> None:+ result = _driver().run(+ "UserProjects",+ route(+ {+ "/api_v2/users.json": [_page("users", [{"id": 4561}, {"id": 4562}]), _page("users", [])],+ "/api_v2/users/4561/projects.json": [+ _page("projects", [{"id": 1000, "name": "Site A"}, {"id": 1001, "name": "Site B"}]),+ _page("projects", []),+ ],+ "/api_v2/users/4562/projects.json": [+ _page("projects", [{"id": 1000, "name": "Site A"}]),+ _page("projects", []),+ ],+ }+ ),+ should_use_incremental_field=True,+ db_incremental_field_last_value=datetime(2024, 1, 1, tzinfo=UTC),+ incremental_field="created_at",+ )++ # Project 1000 appears once per member, so the key must include the user.+ assert result.raised is None+ assert result.response is not None and result.response.primary_keys == ["user_id", "id"]+ assert sorted((row["user_id"], row["id"]) for row in result.rows) == [(4561, 1000), (4561, 1001), (4562, 1000)]+ assert result.params("created_since") == [None] * len(result.requests)+@patch("products.warehouse_sources.backend.temporal.data_imports.sources.bugherd.bugherd.build_dependent_resource")def test_tasks_fanout_seeds_resume_state_and_wires_hook(self, mock_build) -> None:mock_build.return_value = iter([])@@ -171,9 +198,16 @@ class TestBugherdFanout:class TestBugherdChainedFanout:- def test_walks_projects_then_tasks_then_comments(self) -> None:+ @parameterized.expand(+ [+ ("TaskComments", "comments", {"id": 8801, "text": "reproduced"}),+ ("TaskAttachments", "attachments", {"id": 331, "file_name": "screenshot.png"}),+ ]+ )+ def test_walks_projects_then_tasks_then_child(self, endpoint: str, resource: str, row: dict[str, Any]) -> None:+ child_path = f"/api_v2/projects/1000/tasks/98765/{resource}.json"result = _driver().run(- "TaskComments",+ endpoint,route({"/api_v2/projects.json": [_page("projects", [{"id": 1000}]), _page("projects", [])],@@ -181,17 +215,15 @@ class TestBugherdChainedFanout:_page("tasks", [{"id": 98765, "project_id": 1000}]),_page("tasks", []),],- "/api_v2/projects/1000/tasks/98765/comments.json": [- _page("comments", [{"id": 8801, "text": "reproduced"}]),- _page("comments", []),- ],+ child_path: [_page(resource, [row]), _page(resource, [])],}),)assert result.raised is None- assert result.rows == [{"id": 8801, "text": "reproduced", "task_id": 98765, "project_id": 1000}]- assert "/api_v2/projects/1000/tasks/98765/comments.json" in result.paths+ assert result.rows == [{**row, "task_id": 98765, "project_id": 1000}]+ assert result.response is not None and result.response.primary_keys == ["task_id", "id"]+ assert child_path in result.pathsdef test_skips_a_task_that_disappeared_between_pages(self) -> None:result = _driver().run(