:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add productboard configuration and relationship tables (#126779)
Tom Owers0f0e3d43fb16
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 78412f5fa43d..efd67dd59521 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9772,14 +9772,14 @@ Note: Single-page reference; v1 covers catalog/shops/products/orders/uploads/web## Productboard — gaps-Today (14): `companies`, `components`, `features`, `initiatives`, `key_results`, `members`, `notes`, `objectives`, `products`, `release_groups`, `releases`, `subfeatures`, `teams`, `users`+Today (17): `companies`, `components`, `entity_configurations`, `entity_relationships`, `features`, `initiatives`, `key_results`, `members`, `note_relationships`, `notes`, `objectives`, `products`, `release_groups`, `releases`, `subfeatures`, `teams`, `users`Diffed against: <https://developer.productboard.com/reference/listentityfieldvalues>-- [ ] `GET /v2/entities/configurations/{type}` — defines every custom field and status for each entity type - the lookup that names the field/status IDs stored on entities (high)+- [x] `GET /v2/entities/configurations/{type}` — defines every custom field and status for each entity type - the lookup that names the field/status IDs stored on entities (high)- [ ] `GET /v2/entities/fields/{fieldId}/values` — actual custom-field and status values per entity (e.g. status per feature); without it the entity rows carry no field data (high)-- [ ] `GET /v2/entities/{id}/relationships` — the join table linking features to subfeatures, components, products, objectives and initiatives (high)-- [ ] `GET /v2/notes/{id}/relationships` — links each piece of customer feedback to the feature/company it belongs to - the core insight-to-feature join (high)+- [x] `GET /v2/entities/{id}/relationships` — the join table linking features to subfeatures, components, products, objectives and initiatives (high)+- [x] `GET /v2/notes/{id}/relationships` — links each piece of customer feedback to the feature/company it belongs to - the core insight-to-feature join (high)- [ ] `GET /v2/teams/{id}/members` — team membership join table for the teams and members already synced (high)- [ ] `GET /v2/entities/{id}/score` — customer/user impact score per feature - a headline prioritization metric (medium)- [ ] `GET /v2/analytics/member-activities` — per-member activity events for adoption and engagement reporting (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/fanout.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/fanout.pyindex f64aa4b892c5..be768d8586db 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/fanout.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/rest_source/fanout.py@@ -315,6 +315,7 @@ def build_dependent_resource(source_id: str | None = None,use_warehouse_parent: bool = False,parent_snapshot_at: datetime | None = None,+ resume_parent_pagination: bool = False,) -> Iterable[Any]:parent_config = endpoint_configs[fanout.parent_name]child_config = endpoint_configs[child_endpoint]@@ -357,6 +358,8 @@ def build_dependent_resource(incremental_field=incremental_field,incremental_config_factory=incremental_config_factory,)+ if resume_parent_pagination:+ child_resource["resume_parent_pagination"] = Trueconfig: RESTAPIConfig = {"client": client_config,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/canonical_descriptions.pyindex f40ce85db18e..0c51d2cfada0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/canonical_descriptions.py@@ -24,6 +24,15 @@ def _entity_columns(**overrides: str) -> dict[str, str]:return {**_ENTITY_COLUMNS, **overrides}+# Fields shared by relationship rows from the entity and note relationship endpoints.+_RELATIONSHIP_COLUMNS = {+ "type": "The relationship type (e.g. parent, child, link, customer).",+ "target_id": "ID of the entity at the other end of the relationship.",+ "target_type": "Type of the entity at the other end of the relationship (e.g. feature, company, user).",+ "fields": "Field values the relationship carries, keyed by field ID.",+}++CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"features": {"description": "A feature in Productboard — a unit of product work being planned or built.",@@ -151,4 +160,31 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updatedAt": "Time at which the team was last updated.",},},+ "entity_configurations": {+ "description": "The configuration of one Productboard entity type: its fields, including custom fields and statuses, and the relationships it allows.",+ "docs_url": "https://developer.productboard.com/reference/listentityconfigurations",+ "columns": {+ "type": "The entity type this configuration describes (e.g. feature, component, initiative).",+ "fields": "The fields of the entity type, keyed by field ID (a UUID for custom fields). Each entry gives the field's name, type and allowed values.",+ "relationships": "The relationships the entity type allows, keyed by relationship type (parent, child, link).",+ "filters": "The filter parameters the entity list endpoint accepts for this type.",+ },+ },+ "entity_relationships": {+ "description": "One relationship between a Productboard entity and another entity, such as a feature's parent component or a linked objective.",+ "docs_url": "https://developer.productboard.com/reference/listentityrelationships",+ "columns": {+ "entity_id": "ID of the entity the relationship belongs to.",+ "entity_type": "Type of the entity the relationship belongs to.",+ **_RELATIONSHIP_COLUMNS,+ },+ },+ "note_relationships": {+ "description": "One relationship between a Productboard note and its customer (user or company) or a linked product entity such as a feature.",+ "docs_url": "https://developer.productboard.com/reference/listnoterelationships",+ "columns": {+ "note_id": "ID of the note the relationship belongs to.",+ **_RELATIONSHIP_COLUMNS,+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/productboard.py b/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/productboard.pyindex ae15920591ba..6bfa12e27abc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/productboard.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/productboard.py@@ -1,29 +1,38 @@import dataclassesfrom datetime import UTC, date, datetime-from typing import Any, Optional+from typing import Any, Optional, castfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (JSONResponsePaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponsefrom products.warehouse_sources.backend.temporal.data_imports.sources.productboard.settings import (+ FANOUT_PARENT_ENDPOINTS,PRODUCTBOARD_ENDPOINTS,)PRODUCTBOARD_BASE_URL = "https://api.productboard.com/v2"[email protected](frozen=True)class ProductboardResumeConfig:# Full URL of the next unfetched page — Productboard returns it in the `links.next` body field.- next_url: str+ next_url: Optional[str] = None+ # Fan-out tables checkpoint the shape `build_dependent_resource` emits: which parents finished+ # and where the current one stopped.+ fanout_state: Optional[dict[str, Any]] = Nonedef _non_secret_headers() -> dict[str, str]:@@ -72,6 +81,14 @@ def _build_initial_params(return params+def _flatten_relationship_target(row: dict[str, Any]) -> dict[str, Any]:+ # The target id is part of the primary key, so it has to be a top-level column.+ target = row.pop("target", None) or {}+ row["target_id"] = target.get("id")+ row["target_type"] = target.get("type")+ return row++def productboard_source(access_token: str,endpoint: str,@@ -88,53 +105,82 @@ def productboard_source(endpoint, should_use_incremental_field, db_incremental_field_last_value, incremental_field)- rest_config: RESTAPIConfig = {- "client": {- "base_url": PRODUCTBOARD_BASE_URL,- "headers": _non_secret_headers(),- "auth": {"type": "bearer", "token": access_token},- # Next-page URL lives in the response body under `links.next`; absence of it ends the sync.- "paginator": JSONResponsePaginator(next_url_path="links.next"),- },- "resources": [- {- "name": endpoint,- "endpoint": {- "path": config.path,- "params": params,- "data_selector": "data",- },- }- ],+ client_config: ClientConfig = {+ "base_url": PRODUCTBOARD_BASE_URL,+ "headers": _non_secret_headers(),+ "auth": {"type": "bearer", "token": access_token},+ # `links.next` comes from the response body, so pin every request to the API host and refuse+ # redirects to keep the bearer token from being sent anywhere else.+ "allowed_hosts": [],+ "allow_redirects": False,+ # Next-page URL lives in the response body under `links.next`; absence of it ends the sync.+ "paginator": JSONResponsePaginator(next_url_path="links.next"),}- initial_paginator_state: Optional[dict[str, Any]] = None- if resumable_source_manager.can_resume():- resume = resumable_source_manager.load_state()- if resume is not None:- initial_paginator_state = {"next_url": resume.next_url}-- def save_checkpoint(state: Optional[dict[str, Any]]) -> None:- # Persist only while a next page remains; save AFTER a page is yielded so a crash re-yields- # the last page (merge dedupes on primary key) rather than skipping it.- if state and state.get("next_url"):- resumable_source_manager.save_state(ProductboardResumeConfig(next_url=state["next_url"]))-- # The incremental watermark is applied by `_build_initial_params` as a Productboard-specific server- # filter param, so the framework's generic incremental injection is not used here.- resource = rest_api_resource(- rest_config,- team_id,- job_id,- None,- resume_hook=save_checkpoint,- initial_paginator_state=initial_paginator_state,- )+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None++ resource: Resource+ if config.fanout is not None:++ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:+ if state:+ resumable_source_manager.save_state(ProductboardResumeConfig(fanout_state=state))++ fanout_resource = build_dependent_resource(+ endpoint_configs={**PRODUCTBOARD_ENDPOINTS, **FANOUT_PARENT_ENDPOINTS},+ child_endpoint=endpoint,+ fanout=config.fanout,+ client_config=client_config,+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=None,+ page_size_param=None,+ parent_endpoint_extra={"data_selector": "data"},+ child_endpoint_extra={"data_selector": "data"},+ resume_hook=save_fanout_checkpoint,+ initial_paginator_state=resume.fanout_state if resume else None,+ # Saves the parent page too, so the checkpoint keeps two pages of finished parents rather+ # than every parent of a large entity or note listing.+ resume_parent_pagination=True,+ )+ resource = cast(Resource, fanout_resource).add_map(_flatten_relationship_target)+ else:+ rest_config: RESTAPIConfig = {+ "client": client_config,+ "resources": [+ {+ "name": endpoint,+ "endpoint": {+ "path": config.path,+ "params": params,+ "data_selector": "data",+ },+ }+ ],+ }++ def save_checkpoint(state: Optional[dict[str, Any]]) -> None:+ # Persist only while a next page remains; save AFTER a page is yielded so a crash re-yields+ # the last page (merge dedupes on primary key) rather than skipping it.+ if state and state.get("next_url"):+ resumable_source_manager.save_state(ProductboardResumeConfig(next_url=state["next_url"]))++ # The incremental watermark is applied by `_build_initial_params` as a Productboard-specific server+ # filter param, so the framework's generic incremental injection is not used here.+ resource = rest_api_resource(+ rest_config,+ team_id,+ job_id,+ None,+ resume_hook=save_checkpoint,+ initial_paginator_state={"next_url": resume.next_url} if resume and resume.next_url else None,+ )return SourceResponse(name=endpoint,items=lambda: resource,- primary_keys=[config.primary_key],+ primary_keys=config.primary_keys,sort_mode=config.sort_mode,partition_count=1,partition_size=1,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/settings.pyindex 04787085e32e..e63de4285452 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/settings.py@@ -1,18 +1,21 @@from dataclasses import dataclass, fieldfrom typing import Optional+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SortModefrom products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-@dataclass+@dataclass(frozen=True)class ProductboardEndpointConfig:name: strpath: str# Set for endpoints served by the generic `/entities` resource; the value is the# `type[]` query param Productboard expects (e.g. "feature", "component").entity_type: Optional[str] = None- primary_key: str = "id"+ primary_keys: list[str] = field(default_factory=lambda: ["id"])# Stable, immutable datetime field used for partitioning. Never use `updatedAt`.partition_key: Optional[str] = Nonesort_mode: SortMode = "asc"@@ -21,6 +24,10 @@ class ProductboardEndpointConfig:# actually filters on it (only the notes endpoint exposes such filters).incremental_param_map: dict[str, str] = field(default_factory=dict)default_incremental_field: Optional[str] = None+ fanout: Optional[DependentEndpointConfig] = None+ # Productboard sets the page size server-side; the fan-out helper only reads this when it is+ # given a page-size param, which this source never passes.+ page_size: int = 0@propertydef supports_incremental(self) -> bool:@@ -60,6 +67,26 @@ def _entity_endpoint(name: str, entity_type: str) -> ProductboardEndpointConfig:)+# Relationship rows have no id of their own: a relationship is unique per source, type and target.+# A parent deleted between the parent listing and its child fetch 404s, which is not an error here.+def _relationship_endpoint(+ name: str, path: str, parent_name: str, parent_columns: dict[str, str]+) -> ProductboardEndpointConfig:+ return ProductboardEndpointConfig(+ name=name,+ path=path,+ primary_keys=[parent_columns["id"], "type", "target_id"],+ fanout=DependentEndpointConfig(+ parent_name=parent_name,+ resolve_param="id",+ resolve_field="id",+ include_from_parent=list(parent_columns.keys()),+ parent_field_renames=parent_columns,+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ )++PRODUCTBOARD_ENDPOINTS: dict[str, ProductboardEndpointConfig] = {"features": _entity_endpoint("features", "feature"),"subfeatures": _entity_endpoint("subfeatures", "subfeature"),@@ -76,6 +103,28 @@ PRODUCTBOARD_ENDPOINTS: dict[str, ProductboardEndpointConfig] = {# Members carry no documented top-level timestamp, so no partitioning."members": ProductboardEndpointConfig(name="members", path="/members"),"teams": ProductboardEndpointConfig(name="teams", path="/teams", partition_key="createdAt"),+ # One row per entity type, defining its fields (custom fields, statuses and their options) and+ # allowed relationships. Keyed by type, since the configurations carry no id.+ "entity_configurations": ProductboardEndpointConfig(+ name="entity_configurations", path="/entities/configurations", primary_keys=["type"]+ ),+ "entity_relationships": _relationship_endpoint(+ "entity_relationships",+ "/entities/{id}/relationships",+ parent_name="entities",+ parent_columns={"id": "entity_id", "type": "entity_type"},+ ),+ "note_relationships": _relationship_endpoint(+ "note_relationships",+ "/notes/{id}/relationships",+ parent_name="notes",+ parent_columns={"id": "note_id"},+ ),+}++# Fan-out parents that are not tables themselves. `entities` lists every entity type in one walk.+FANOUT_PARENT_ENDPOINTS: dict[str, ProductboardEndpointConfig] = {+ "entities": ProductboardEndpointConfig(name="entities", path="/entities"),}ENDPOINTS = tuple(PRODUCTBOARD_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/source.pyindex 4a787b038aac..7b8777969487 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/source.py@@ -29,6 +29,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.productboa)from products.warehouse_sources.backend.temporal.data_imports.sources.productboard.settings import (ENDPOINTS,+ FANOUT_PARENT_ENDPOINTS,INCREMENTAL_FIELDS,PRODUCTBOARD_ENDPOINTS,)@@ -42,6 +43,9 @@ def _probe_path(schema_name: Optional[str]) -> str:return "/members"config = PRODUCTBOARD_ENDPOINTS[schema_name]+ if config.fanout is not None:+ # Child paths need a parent id; the parent listing needs the same scope.+ return {**PRODUCTBOARD_ENDPOINTS, **FANOUT_PARENT_ENDPOINTS}[config.fanout.parent_name].pathif config.entity_type:return f"/entities?type[]={config.entity_type}"return config.pathdiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/tests/test_productboard.py b/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/tests/test_productboard.pyindex 292ba4a13344..7ef5ef6b6869 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/tests/test_productboard.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/tests/test_productboard.py@@ -105,3 +105,56 @@ class TestPagination:)assert result.queries[0] == {"updatedFrom": ("2023-10-01T10:00:00Z",)}+++class TestRelationshipFanout:+ def test_entity_relationships_walk_every_entity_and_child_page(self):+ child_page2_url = "https://api.productboard.com/v2/entities/e1/relationships?pageCursor=c2"++ result = _driver().run(+ "entity_relationships",+ [+ _page({"data": [{"id": "e1", "type": "feature"}, {"id": "e2", "type": "component"}], "links": {}}),+ _page(+ {+ "data": [{"type": "parent", "target": {"id": "e2", "type": "component", "links": {}}}],+ "links": {"next": child_page2_url},+ }+ ),+ _page({"data": [{"type": "link", "target": {"id": "o1", "type": "objective"}}], "links": {}}),+ _page({"data": [{"type": "child", "target": {"id": "e1", "type": "feature"}}], "links": {}}),+ ],+ )++ assert result.raised is None+ assert result.urls == [+ "https://api.productboard.com/v2/entities",+ "https://api.productboard.com/v2/entities/e1/relationships",+ child_page2_url,+ "https://api.productboard.com/v2/entities/e2/relationships",+ ]+ assert [+ (row["entity_id"], row["entity_type"], row["type"], row["target_id"], row["target_type"])+ for row in result.rows+ ] == [+ ("e1", "feature", "parent", "e2", "component"),+ ("e1", "feature", "link", "o1", "objective"),+ ("e2", "component", "child", "e1", "feature"),+ ]+ assert all("target" not in row for row in result.rows)+ # The checkpoint tracks the parent page, so it does not keep every finished entity.+ assert result.saved_states and all("parent" in (state.fanout_state or {}) for state in result.saved_states)++ def test_note_relationships_skip_a_note_deleted_after_listing(self):+ result = _driver().run(+ "note_relationships",+ [+ _page({"data": [{"id": "n1"}, {"id": "n2"}], "links": {}}),+ ScriptedResponse(status=404, json={"errors": [{"code": "not_found"}]}),+ _page({"data": [{"type": "customer", "target": {"id": "c1", "type": "company"}}], "links": {}}),+ ],+ )++ assert result.raised is None+ assert result.paths == ["/v2/notes", "/v2/notes/n1/relationships", "/v2/notes/n2/relationships"]+ assert [(row["note_id"], row["type"], row["target_id"]) for row in result.rows] == [("n2", "customer", "c1")]diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/tests/test_productboard_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/tests/test_productboard_source.pyindex d6b20413d078..349e747164b3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/tests/test_productboard_source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/productboard/tests/test_productboard_source.py@@ -48,6 +48,9 @@ class TestProductboardSource:("notes", "/notes"),("members", "/members"),("teams", "/teams"),+ # Fan-out tables probe their parent listing, since the child path needs a parent id.+ ("entity_relationships", "/entities"),+ ("note_relationships", "/notes"),],)def test_probe_path(self, schema_name, expected_path):