Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 4 missing tables to the Bugherd source (#119941)

Tom Owers16a8123b6929
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index cff80bf9fc17..cd80937daaad 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -1937,9 +1937,9 @@ Diffed against: <https://docs.bugherd.com/api/openapi.yaml>
- [x] `GET /api_v2/projects/{project_id}/columns.json` — lookup resolving the board column/status a task sits in (high)
- [x] `GET /api_v2/projects/{project_id}/tasks/archive.json` — archived (closed-out) tasks live behind a separate endpoint, so completed work is entirely missing from the tasks table (high)
- [x] `GET /api_v2/projects/{project_id}/tasks/feedback.json` — the unsorted feedback inbox — tasks not yet promoted to the board (medium)
-- [ ] `GET /api_v2/projects/{project_id}/tasks/{task_id}/attachments.json` — screenshots and files attached to a task, joinable to tasks (medium)
-- [ ] `GET /api_v2/users/{user_id}/projects.json` — membership join between users and projects; neither existing table carries the relationship (medium)
-- [ ] `GET /api_v2/users/members.json and GET /api_v2/users/guests.json` — splits the flat users list into members vs guests, which is how BugHerd seats are licensed (medium)
+- [x] `GET /api_v2/projects/{project_id}/tasks/{task_id}/attachments.json` — screenshots and files attached to a task, joinable to tasks (medium)
+- [x] `GET /api_v2/users/{user_id}/projects.json` — membership join between users and projects; neither existing table carries the relationship (medium)
+- [x] `GET /api_v2/users/members.json and GET /api_v2/users/guests.json` — splits the flat users list into members vs guests, which is how BugHerd seats are licensed (medium)
- [ ] `GET /api_v2/projects/active.json` — distinguishes active from archived projects without inspecting each project (low)
Note: docs.bugherd.com/api is a Scalar shell; the real spec is at /api/openapi.yaml (70KB, 27 paths). Note that the connector's Tasks stream hits /api_v2/projects/{id}/tasks.json, which is a separate endpoint from the archived and feedback task lists — archived tasks are therefore not synced at all, not merely filtered.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/canonical_descriptions.py
index 5dbbfcef369e..9d6f90e0ac8e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/canonical_descriptions.py
@@ -33,6 +33,14 @@ _TASK_COLUMNS = {
"admin_link": "Task board URL for the task; login required.",
}
+# The members and guests lists return the same User object as the full users list.
+_USER_COLUMNS = {
+ "id": "Unique identifier for the user.",
+ "email": "The user's email address.",
+ "display_name": "The user's display name.",
+ "avatar_url": "URL of the user's avatar image.",
+}
+
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"Organization": {
"description": "Top-level details about your BugHerd organization.",
@@ -45,12 +53,17 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"Users": {
"description": "Team members and guests who belong to your BugHerd organization.",
"docs_url": _API_DOCS_URL,
- "columns": {
- "id": "Unique identifier for the user.",
- "email": "The user's email address.",
- "display_name": "The user's display name.",
- "avatar_url": "URL of the user's avatar image.",
- },
+ "columns": _USER_COLUMNS,
+ },
+ "Members": {
+ "description": "Team members (non-guest users) in your BugHerd organization.",
+ "docs_url": _API_DOCS_URL,
+ "columns": _USER_COLUMNS,
+ },
+ "Guests": {
+ "description": "Guest users in your BugHerd organization: external collaborators invited to specific projects.",
+ "docs_url": _API_DOCS_URL,
+ "columns": _USER_COLUMNS,
},
"Projects": {
"description": "Websites/projects tracked for visual feedback and bug reporting in BugHerd.",
@@ -112,4 +125,27 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"is_private": "Whether the comment is visible only to team members, not guests/clients.",
},
},
+ "TaskAttachments": {
+ "description": "Screenshots and other files attached to a task.",
+ "docs_url": _API_DOCS_URL,
+ "columns": {
+ "id": "Identifier for the attachment, unique within its task.",
+ "task_id": "ID of the task the file is attached to.",
+ "project_id": "ID of the project the task belongs to.",
+ "file_name": "Name of the attached file.",
+ "url": "URL of the attached file.",
+ "created_at": "Timestamp when the file was attached.",
+ },
+ },
+ "UserProjects": {
+ "description": "Project memberships: one row for each project a user belongs to.",
+ "docs_url": _API_DOCS_URL,
+ "columns": {
+ "user_id": "ID of the user who belongs to the project.",
+ "id": "ID of the project the user belongs to.",
+ "name": "The project's display name.",
+ "created_at": "Timestamp when the project was created.",
+ "is_active": "Whether the project is active.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/settings.py
index 296a3517763c..ba861ee6b673 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/settings.py
@@ -17,6 +17,14 @@ PROJECT_ID_FANOUT = DependentEndpointConfig(
parent_field_renames={"id": "project_id"},
)
+USER_ID_FANOUT = DependentEndpointConfig(
+ parent_name="Users",
+ resolve_param="user_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "user_id"},
+)
+
@dataclass(frozen=True)
class BugherdChainedFanoutConfig:
@@ -35,7 +43,7 @@ class BugherdChainedFanoutConfig:
parent_field_renames: dict[str, str]
-TASK_COMMENTS_FANOUT = BugherdChainedFanoutConfig(
+TASK_ID_FANOUT = BugherdChainedFanoutConfig(
parent_name="Tasks",
resolve_fields={"project_id": "project_id", "task_id": "id"},
include_from_parent=["id", "project_id"],
@@ -82,6 +90,19 @@ BUGHERD_ENDPOINTS: dict[str, BugherdEndpointConfig] = {
path="/api_v2/users.json",
data_selector="users",
),
+ # The same User objects as `Users`, split by seat type: team members versus guests.
+ # The User schema carries no member/guest flag, so these lists are the only place
+ # the distinction appears.
+ "Members": BugherdEndpointConfig(
+ name="Members",
+ path="/api_v2/users/members.json",
+ data_selector="users",
+ ),
+ "Guests": BugherdEndpointConfig(
+ name="Guests",
+ path="/api_v2/users/guests.json",
+ data_selector="users",
+ ),
"Projects": BugherdEndpointConfig(
name="Projects",
path="/api_v2/projects.json",
@@ -155,7 +176,30 @@ BUGHERD_ENDPOINTS: dict[str, BugherdEndpointConfig] = {
# so treat the ID as task-scoped rather than global.
primary_key=["task_id", "id"],
partition_key="created_at",
- chained_fanout=TASK_COMMENTS_FANOUT,
+ chained_fanout=TASK_ID_FANOUT,
+ ),
+ # Screenshots and files on a task. Like comments, the endpoint has no timestamp
+ # filter, so full refresh.
+ "TaskAttachments": BugherdEndpointConfig(
+ name="TaskAttachments",
+ path="/api_v2/projects/{project_id}/tasks/{task_id}/attachments.json",
+ data_selector="attachments",
+ # Addressed by project, task and attachment ID, so the ID is task-scoped.
+ primary_key=["task_id", "id"],
+ partition_key="created_at",
+ chained_fanout=TASK_ID_FANOUT,
+ ),
+ # The user-to-project membership join: one row per project a user belongs to. Each
+ # row is a full Project object, so `id` is the project ID. The `created_since`
+ # filter only reports projects created after the watermark, which would miss a user
+ # added to an older project, so this stays full refresh.
+ "UserProjects": BugherdEndpointConfig(
+ name="UserProjects",
+ path="/api_v2/users/{user_id}/projects.json",
+ data_selector="projects",
+ primary_key=["user_id", "id"],
+ partition_key="created_at",
+ fanout=USER_ID_FANOUT,
),
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/tests/test_bugherd.py b/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/tests/test_bugherd.py
index 75fc6fe79923..2bc0659856ac 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/tests/test_bugherd.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/bugherd/tests/test_bugherd.py
@@ -154,6 +154,33 @@ class TestBugherdFanout:
assert result.params("created_since") == [None, "2024-01-01T00:00:00Z", "2024-01-01T00:00:00Z", None]
assert result.params("updated_since") == [None, None, None, None]
+ def test_user_projects_fans_out_over_users_and_stays_full_refresh(self) -> None:
+ result = _driver().run(
+ "UserProjects",
+ route(
+ {
+ "/api_v2/users.json": [_page("users", [{"id": 4561}, {"id": 4562}]), _page("users", [])],
+ "/api_v2/users/4561/projects.json": [
+ _page("projects", [{"id": 1000, "name": "Site A"}, {"id": 1001, "name": "Site B"}]),
+ _page("projects", []),
+ ],
+ "/api_v2/users/4562/projects.json": [
+ _page("projects", [{"id": 1000, "name": "Site A"}]),
+ _page("projects", []),
+ ],
+ }
+ ),
+ should_use_incremental_field=True,
+ db_incremental_field_last_value=datetime(2024, 1, 1, tzinfo=UTC),
+ incremental_field="created_at",
+ )
+
+ # Project 1000 appears once per member, so the key must include the user.
+ assert result.raised is None
+ assert result.response is not None and result.response.primary_keys == ["user_id", "id"]
+ assert sorted((row["user_id"], row["id"]) for row in result.rows) == [(4561, 1000), (4561, 1001), (4562, 1000)]
+ assert result.params("created_since") == [None] * len(result.requests)
+
@patch("products.warehouse_sources.backend.temporal.data_imports.sources.bugherd.bugherd.build_dependent_resource")
def test_tasks_fanout_seeds_resume_state_and_wires_hook(self, mock_build) -> None:
mock_build.return_value = iter([])
@@ -171,9 +198,16 @@ class TestBugherdFanout:
class TestBugherdChainedFanout:
- def test_walks_projects_then_tasks_then_comments(self) -> None:
+ @parameterized.expand(
+ [
+ ("TaskComments", "comments", {"id": 8801, "text": "reproduced"}),
+ ("TaskAttachments", "attachments", {"id": 331, "file_name": "screenshot.png"}),
+ ]
+ )
+ def test_walks_projects_then_tasks_then_child(self, endpoint: str, resource: str, row: dict[str, Any]) -> None:
+ child_path = f"/api_v2/projects/1000/tasks/98765/{resource}.json"
result = _driver().run(
- "TaskComments",
+ endpoint,
route(
{
"/api_v2/projects.json": [_page("projects", [{"id": 1000}]), _page("projects", [])],
@@ -181,17 +215,15 @@ class TestBugherdChainedFanout:
_page("tasks", [{"id": 98765, "project_id": 1000}]),
_page("tasks", []),
],
- "/api_v2/projects/1000/tasks/98765/comments.json": [
- _page("comments", [{"id": 8801, "text": "reproduced"}]),
- _page("comments", []),
- ],
+ child_path: [_page(resource, [row]), _page(resource, [])],
}
),
)
assert result.raised is None
- assert result.rows == [{"id": 8801, "text": "reproduced", "task_id": 98765, "project_id": 1000}]
- assert "/api_v2/projects/1000/tasks/98765/comments.json" in result.paths
+ assert result.rows == [{**row, "task_id": 98765, "project_id": 1000}]
+ assert result.response is not None and result.response.primary_keys == ["task_id", "id"]
+ assert child_path in result.paths
def test_skips_a_task_that_disappeared_between_pages(self) -> None:
result = _driver().run(