:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 4 missing tables to the Phyllo source (#123803)
Tom Owers830501a421d8
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex e48203001f30..f0952a12d897 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -867,7 +867,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/personio/personio.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pganalyze/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/phyllo.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/pingdom.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pingdom/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/pinterest_ads/pinterest_ads.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 2d0d9ecb5248..12f46d738b10 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9379,10 +9379,10 @@ Today (7): `accounts`, `income_payouts`, `income_transactions`, `profiles`, `socDiffed against: <https://rapidapi.com/phyllo-phyllo-default/api/phyllo-apis-v1>-- [ ] `/v1/audience` — audience demographics and follower breakdowns - the headline output of the Identity & Audience product (high)-- [ ] `/v1/social/comments` — comment-level engagement for the contents we already sync; the Comments API is one of Phyllo's three product lines (high)-- [ ] `/v1/social/content-groups` — stories/albums/series groupings that contents belong to; needed to roll engagement up above the individual post (medium)-- [ ] `/v1/commerce/income/transactions` — commerce-platform earnings; the synced income_transactions table only covers the social income path (medium)+- [x] `/v1/audience` — audience demographics and follower breakdowns - the headline output of the Identity & Audience product (high)+- [x] `/v1/social/comments` — comment-level engagement for the contents we already sync; the Comments API is one of Phyllo's three product lines (high)+- [x] `/v1/social/content-groups` — stories/albums/series groupings that contents belong to; needed to roll engagement up above the individual post (medium)+- [x] `/v1/commerce/income/transactions` — commerce-platform earnings; the synced income_transactions table only covers the social income path (medium)- [ ] `/v1/commerce/income/payouts` — commerce payout records, same coverage gap as commerce transactions (medium)- [ ] `/v1/commerce/income/balances` — current and pending balances per connected commerce account, not derivable from payouts alone (medium)- [ ] `/v1/media/activity/contents` — media-activity (listening/watching) items for accounts on media platforms (medium)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/canonical_descriptions.pyindex 3b0e12091579..6b021b4a85bc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/canonical_descriptions.py@@ -122,4 +122,83 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updated_at": "When the payout record was last updated.",},},+ "audience": {+ "description": "Lifetime audience demographics of a connected account: the share of the audience by country, city, gender and age range.",+ "docs_url": "https://docs.getphyllo.com",+ "columns": {+ "id": "The unique ID of the audience demographics object.",+ "account": "The connected account the audience belongs to.",+ "user": "The Phyllo user the audience belongs to.",+ "work_platform": "The platform the audience is on.",+ "countries": "Audience share by country, as ISO 3166-1 alpha-2 codes with percentage values.",+ "cities": "Audience share by city, as city names with percentage values.",+ "gender_age_distribution": "Audience share by gender and age range, as percentage values.",+ "created_at": "When the audience record was created in Phyllo.",+ "updated_at": "When the audience record was last updated.",+ },+ },+ "social_comments": {+ "description": "A comment on a content item published by a connected account.",+ "docs_url": "https://docs.getphyllo.com",+ "columns": {+ "id": "The unique ID of the comment.",+ "account": "The connected account that published the commented content.",+ "user": "The Phyllo user the comment belongs to.",+ "work_platform": "The platform the comment was posted on.",+ "content": "The content item the comment was posted on.",+ "external_id": "The platform's own identifier for the comment.",+ "text": "The text of the comment.",+ "commenter_username": "The username of the commenter's profile.",+ "commenter_display_name": "The full display name of the commenter's profile.",+ "commenter_id": "The platform's identifier for the commenter's profile.",+ "commenter_profile_url": "URL of the commenter's profile on the platform.",+ "like_count": "The total number of likes on the comment.",+ "reply_count": "The total number of replies to the comment.",+ "published_at": "When the comment was published on the platform.",+ "created_at": "When the comment record was created in Phyllo.",+ "updated_at": "When the comment record was last updated.",+ },+ },+ "social_content_groups": {+ "description": "A group of content items published by a connected account, such as a story, album or playlist, with its engagement metrics.",+ "docs_url": "https://docs.getphyllo.com",+ "columns": {+ "id": "The unique ID of the content group.",+ "account": "The connected account that published the content group.",+ "user": "The Phyllo user the content group belongs to.",+ "work_platform": "The platform the content group was published on.",+ "external_id": "The platform's own identifier for the content group.",+ "title": "The title of the content group.",+ "format": "The media format of the content group.",+ "type": "The platform-specific content type.",+ "url": "URL of the content group on the platform, such as a YouTube playlist URL.",+ "description": "The description of the content group.",+ "visibility": "The visibility of the content group.",+ "engagement": "Engagement metrics of the content group.",+ "item_count": "The number of content items in the content group.",+ "published_at": "When the content group was published on the platform.",+ "created_at": "When the content group record was created in Phyllo.",+ "updated_at": "When the content group record was last updated.",+ },+ },+ "commerce_income_transactions": {+ "description": "A transaction of a connected e-commerce account, such as a sale or a refund.",+ "docs_url": "https://docs.getphyllo.com",+ "columns": {+ "id": "The unique ID of the transaction.",+ "account": "The connected account the transaction belongs to.",+ "user": "The Phyllo user the transaction belongs to.",+ "work_platform": "The e-commerce platform the transaction occurred on.",+ "external_id": "The platform's own identifier for the transaction.",+ "amount": "The monetary amount of the transaction.",+ "currency": "The ISO 4217 currency code of the transaction amount.",+ "type": "The type of the transaction (SALE, REFUND, CANCELLED, VOID or OTHER).",+ "status": "The status of the transaction on the platform.",+ "transaction_at": "When the transaction occurred on the platform.",+ "platform_profile_id": "The unique profile ID of the user on the platform.",+ "platform_profile_name": "The profile name of the user on the platform.",+ "created_at": "When the transaction record was created in Phyllo.",+ "updated_at": "When the transaction record was last updated.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/phyllo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/phyllo.pyindex 579383ff7f6c..4a5da48406fb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/phyllo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/phyllo.py@@ -1,4 +1,5 @@import dataclasses+from datetime import UTC, date, datetimefrom typing import Any, Optionalfrom requests.auth import HTTPBasicAuth@@ -8,10 +9,17 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.resRESTAPIConfig,rest_api_resources,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.config_setup import (+ make_parent_key_name,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (OffsetPaginator,+ SinglePagePaginator,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (+ Endpoint,+ EndpointResource,)-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResourcefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse@@ -26,6 +34,9 @@ PHYLLO_BASE_URLS: dict[str, str] = {# Documented maximum page size for Phyllo list endpoints.PAGE_SIZE = 100ACCOUNTS_PATH = "/v1/accounts"+# Phyllo documents no lower bound for a required date window, so a full refresh starts far enough+# back to cover the whole history of an account.+DATE_WINDOW_START = date(2000, 1, 1)# Cheap list endpoint used to confirm a client ID/secret pair is genuine. Phyllo credentials are# environment-wide, so one probe validates access to every endpoint.DEFAULT_PROBE_PATH = "/v1/work-platforms"@@ -54,7 +65,30 @@ def _base_headers() -> dict[str, str]:return {"Accept": "application/json"}-def _build_resources(endpoint: str) -> list[EndpointResource | str]:+def _date_window_params(endpoint: str, today: date) -> dict[str, str]:+ config = PHYLLO_ENDPOINTS[endpoint]+ if config.date_window_params is None:+ return {}++ from_param, to_param = config.date_window_params+ return {from_param: DATE_WINDOW_START.isoformat(), to_param: today.isoformat()}+++def _child_endpoint(path: str, params: dict[str, Any], *, single_object: bool = False) -> Endpoint:+ if single_object:+ return {"path": path, "params": params, "data_selector": "$", "paginator": SinglePagePaginator()}+ return {+ "path": path,+ "params": params,+ "data_selector": "data",+ # Dependent resources can't classify a malformed 200 body as retryable, so fail loud+ # instead of silently syncing 0 rows if the response shape changes.+ "data_selector_required": True,+ "paginator": OffsetPaginator(limit=PAGE_SIZE, total_path=None),+ }+++def _build_resources(endpoint: str, extra_params: dict[str, str]) -> list[EndpointResource | str]:config = PHYLLO_ENDPOINTS[endpoint]if not config.fan_out_by_account:@@ -85,22 +119,54 @@ def _build_resources(endpoint: str) -> list[EndpointResource | str]:"paginator": OffsetPaginator(limit=PAGE_SIZE, total_path=None),},}+ account_id_param = {"type": "resolve", "resource": "accounts", "field": "id"}++ if config.fan_out_by_content:+ # The parent account id is carried onto each content row, so the comments request can bind+ # both ids from one parent row. The injected key is not in the comment rows.+ contents_parent: EndpointResource = {+ "name": "social_contents",+ "endpoint": _child_endpoint(+ f"{PHYLLO_ENDPOINTS['social_contents'].path}?account_id={{account_id}}",+ {"account_id": account_id_param},+ ),+ "include_from_parent": ["id"],+ }+ comments: EndpointResource = {+ "name": endpoint,+ "endpoint": _child_endpoint(+ f"{config.path}?account_id={{account_id}}&content_id={{content_id}}",+ {+ "account_id": {+ "type": "resolve",+ "resource": "social_contents",+ "field": make_parent_key_name("accounts", "id"),+ },+ "content_id": {"type": "resolve", "resource": "social_contents", "field": "id"},+ },+ ),+ "include_from_parent": [],+ }+ return [accounts_parent, contents_parent, comments]+child: EndpointResource = {"name": endpoint,- "endpoint": {- "path": f"{config.path}?account_id={{account_id}}",- "params": {"account_id": {"type": "resolve", "resource": "accounts", "field": "id"}},- "data_selector": "data",- # Dependent resources can't classify a malformed 200 body as retryable, so fail loud- # instead of silently syncing 0 rows if the response shape changes.- "data_selector_required": True,- "paginator": OffsetPaginator(limit=PAGE_SIZE, total_path=None),- },+ "endpoint": _child_endpoint(+ f"{config.path}?account_id={{account_id}}",+ {"account_id": account_id_param, **extra_params},+ single_object=config.single_object,+ ),"include_from_parent": [],}return [accounts_parent, child]+def endpoint_supports_resume(endpoint: str) -> bool:+ # The framework cannot checkpoint the two-level accounts -> contents -> comments fan-out, so a+ # retry of that table starts again from the first account.+ return not PHYLLO_ENDPOINTS[endpoint].fan_out_by_content++def phyllo_source(client_id: str,client_secret: str,@@ -123,11 +189,12 @@ def phyllo_source("paginator": OffsetPaginator(limit=PAGE_SIZE, total_path=None),},"resource_defaults": {},- "resources": _build_resources(endpoint),+ "resources": _build_resources(endpoint, _date_window_params(endpoint, datetime.now(UTC).date())),}+ supports_resume = endpoint_supports_resume(endpoint)initial_paginator_state: Optional[dict[str, Any]] = None- if resumable_source_manager.can_resume():+ if supports_resume and resumable_source_manager.can_resume():resume = resumable_source_manager.load_state()if resume is not None and resume.paginator_state is not None:initial_paginator_state = resume.paginator_state@@ -143,7 +210,7 @@ def phyllo_source(team_id,job_id,db_incremental_field_last_value,- resume_hook=save_checkpoint,+ resume_hook=save_checkpoint if supports_resume else None,initial_paginator_state=initial_paginator_state,)target = next(resource for resource in resources if resource.name == endpoint)@@ -157,6 +224,7 @@ def phyllo_source(partition_count=1,partition_size=1,column_hints=target.column_hints,+ supports_resume=supports_resume,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/settings.pyindex 1703100c0167..63f763ef9059 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/settings.py@@ -1,15 +1,23 @@-from dataclasses import dataclass, field+from dataclasses import field++from posthog.dataclasses import frozenfrom products.warehouse_sources.backend.types import IncrementalField-@dataclass+@frozenclass PhylloEndpointConfig:name: strpath: str# Endpoints keyed off a linked creator account require an `account_id` query param, so we fan# out over the accounts listing and pull each account's rows in turn.fan_out_by_account: bool = False+ # Comments need both `account_id` and `content_id`, so we fan out accounts -> contents -> comments.+ fan_out_by_content: bool = False+ # The endpoint returns one object per account instead of a paginated `data` list.+ single_object: bool = False+ # Query params for a date window that the endpoint requires on every request (from, to).+ date_window_params: tuple[str, str] | None = None# Phyllo object identifiers are platform-generated UUIDs unique across the system (a content# item, transaction, or payout belongs to exactly one account), so `id` is a safe table-wide key.primary_keys: list[str] = field(default_factory=lambda: ["id"])@@ -19,6 +27,9 @@ class PhylloEndpointConfig:# content and income endpoints document from_date/to_date filters, but we could not verify that# they are honored server-side (and no endpoint documents an updated-after cursor), so we# conservatively re-pull each stream every sync (see the implementing-warehouse-sources skill).+# commerce_income_transactions requires its date window, but one table-wide watermark would skip the+# history of an account connected after the first sync, and the status updates of older+# transactions, so it also re-pulls the full window every sync.PHYLLO_ENDPOINTS: dict[str, PhylloEndpointConfig] = {"work_platforms": PhylloEndpointConfig(name="work_platforms", path="/v1/work-platforms"),"users": PhylloEndpointConfig(name="users", path="/v1/users"),@@ -31,6 +42,19 @@ PHYLLO_ENDPOINTS: dict[str, PhylloEndpointConfig] = {name="income_transactions", path="/v1/income/transactions", fan_out_by_account=True),"income_payouts": PhylloEndpointConfig(name="income_payouts", path="/v1/income/payouts", fan_out_by_account=True),+ "audience": PhylloEndpointConfig(name="audience", path="/v1/audience", fan_out_by_account=True, single_object=True),+ "social_comments": PhylloEndpointConfig(+ name="social_comments", path="/v1/social/comments", fan_out_by_account=True, fan_out_by_content=True+ ),+ "social_content_groups": PhylloEndpointConfig(+ name="social_content_groups", path="/v1/social/content-groups", fan_out_by_account=True+ ),+ "commerce_income_transactions": PhylloEndpointConfig(+ name="commerce_income_transactions",+ path="/v1/commerce/income/transactions",+ fan_out_by_account=True,+ date_window_params=("transaction_from_date", "transaction_to_date"),+ ),}ENDPOINTS = tuple(PHYLLO_ENDPOINTS.keys())diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/source.pyindex 11dd7d0eeafc..44a94821ca5a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/source.py@@ -24,6 +24,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typfrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.phyllo import PhylloSourceConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.phyllo.phyllo import (PhylloResumeConfig,+ endpoint_supports_resume,phyllo_source,validate_credentials,)@@ -139,6 +140,16 @@ You can find your client ID and secret in the [Phyllo developer dashboard](https# Credentials are environment-wide, so a single probe validates access to every schema.return validate_credentials(config.client_id, config.client_secret, config.environment)+ def resume_covers_run(+ self,+ *,+ incremental_or_append: bool,+ schema_name: str | None = None,+ ) -> bool:+ # An endpoint that cannot checkpoint restarts from the first account on every attempt, so the+ # larger resumable retry budget would only repeat the whole fan-out.+ return schema_name is None or schema_name not in PHYLLO_ENDPOINTS or endpoint_supports_resume(schema_name)+def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[PhylloResumeConfig]:return ResumableSourceManager[PhylloResumeConfig](inputs, PhylloResumeConfig)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/tests/test_phyllo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/tests/test_phyllo.pyindex 73b3e3b58980..72df311b1dbe 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/tests/test_phyllo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/phyllo/tests/test_phyllo.py@@ -1,5 +1,8 @@+from datetime import UTC, datetimefrom typing import Any+import time_machine+from parameterized import parameterizedfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (@@ -77,6 +80,75 @@ class TestFanOut:assert [request.param("account_id") for request in child_requests] == ["acc_a", "acc_c"]assert [request.param("offset") for request in child_requests] == ["0", "0"]+ def test_comments_bind_account_and_content_ids(self) -> None:+ result = _driver().run(+ "social_comments",+ route(+ {+ "/v1/accounts": [_response([{"id": "acc_a"}, {"id": "acc_b"}])],+ "/v1/social/contents": [+ _response([{"id": "con_1"}, {"id": "con_2"}]),+ _response([{"id": "con_3"}]),+ ],+ "/v1/social/comments": [+ _response([{"id": "cm_1"}]),+ _response([]),+ _response([{"id": "cm_3"}]),+ ],+ }+ ),+ )++ assert result.raised is None+ # The account id carried onto each content row to bind the request must not leak into rows.+ assert result.rows == [{"id": "cm_1"}, {"id": "cm_3"}]+ comment_requests = [request for request in result.requests if request.path == "/v1/social/comments"]+ assert [(request.param("account_id"), request.param("content_id")) for request in comment_requests] == [+ ("acc_a", "con_1"),+ ("acc_a", "con_2"),+ ("acc_b", "con_3"),+ ]++ def test_audience_yields_one_row_per_account(self) -> None:+ result = _driver().run(+ "audience",+ route(+ {+ "/v1/accounts": [_response([{"id": "acc_a"}, {"id": "acc_b"}])],+ "/v1/audience": [+ ScriptedResponse(json={"id": "aud_a", "countries": [{"code": "US", "value": 60.5}]}),+ ScriptedResponse(json={"id": "aud_b", "countries": []}),+ ],+ }+ ),+ )++ assert result.raised is None+ assert [row["id"] for row in result.rows] == ["aud_a", "aud_b"]+ audience_requests = [request for request in result.requests if request.path == "/v1/audience"]+ assert [request.param("account_id") for request in audience_requests] == ["acc_a", "acc_b"]+++class TestCommerceTransactionDateWindow:+ def test_window_covers_full_history(self) -> None:+ with time_machine.travel(datetime(2026, 3, 15, 12, 0, tzinfo=UTC), tick=False):+ result = _driver().run(+ "commerce_income_transactions",+ route(+ {+ "/v1/accounts": [_response([{"id": "acc_a"}])],+ "/v1/commerce/income/transactions": [_response([{"id": "txn_1"}])],+ }+ ),+ )++ assert result.raised is None+ assert result.rows == [{"id": "txn_1"}]+ [request] = [r for r in result.requests if r.path == "/v1/commerce/income/transactions"]+ assert request.param("account_id") == "acc_a"+ assert request.param("transaction_from_date") == "2000-01-01"+ assert request.param("transaction_to_date") == "2026-03-15"+class TestMalformedBody:def test_top_level_persistent_malformed_body_reraises(self) -> None:@@ -133,12 +205,25 @@ class TestPhylloSourceResponse:assert result.response.primary_keys == ["id"]# Phyllo doesn't guarantee ordering or a stable creation timestamp, so we don't partition.assert result.response.partition_mode is None+ # The two-level comments fan-out has no framework checkpoint, so it must not claim resume.+ assert result.response.supports_resume is (endpoint != "social_comments")+ assert PhylloSource().resume_covers_run(incremental_or_append=False, schema_name=endpoint) is (+ endpoint != "social_comments"+ )def test_fan_out_endpoints_are_account_scoped(self) -> None:# These endpoints require an account_id query param; a config regression here would 400 on# every page.fan_out = {name for name, config in PHYLLO_ENDPOINTS.items() if config.fan_out_by_account}- assert fan_out == {"social_contents", "income_transactions", "income_payouts"}+ assert fan_out == {+ "social_contents",+ "income_transactions",+ "income_payouts",+ "audience",+ "social_comments",+ "social_content_groups",+ "commerce_income_transactions",+ }@parameterized.expand([("production", "https://api.getphyllo.com"), ("sandbox", "https://api.sandbox.getphyllo.com")]