:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 1 missing table to the Simplesat source (#120553)
Tom Owersf4a789c4fc75
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 5291efa32790..1677e2dbaffc 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -1029,7 +1029,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/similarweb/similarweb.py1 products/warehouse_sources/backend/temporal/data_imports/sources/simplecast/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/simplecast/simplecast.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/simplesat.py1 products/warehouse_sources/backend/temporal/data_imports/sources/skyvern/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/skyvern/skyvern.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex eccbbcff6374..a13a6718c9e4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -11125,7 +11125,7 @@ Today (4): `answers`, `questions`, `responses`, `surveys`Diffed against: <https://developer.simplesat.io/api/Simplesat%20API%20(v1)%20OpenAPI.yaml>-- [ ] `customers (GET /api/v1/customers)` — lookup table resolving the customer referenced on every synced response, so CSAT/NPS can be sliced by account (high)+- [x] `customers (GET /api/v1/customers)` — lookup table resolving the customer referenced on every synced response, so CSAT/NPS can be sliced by account (high)Note: The v1 OpenAPI defines only six resources: answers, customers, team-members, responses, surveys, questions. PostHog syncs four of them, so coverage is proportionate — only customers is a real miss. team-members exposes GET /api/v1/team-members/{team_member_id} but no list endpoint (the collection path is POST-only), so it cannot be synced as a table without hydrating ids from responses; I did not list it as a gap for that reason.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/canonical_descriptions.pyindex de9e965860ac..c6b5a83c5829 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/canonical_descriptions.py@@ -63,4 +63,21 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"modified": "When the response was last modified.",},},+ "customers": {+ "description": "A customer (survey respondent) in the Simplesat account.",+ "docs_url": "https://developer.simplesat.io/api",+ "columns": {+ "id": "The unique ID of the customer.",+ "external_id": "The customer's ID in an external system, such as a help desk or CRM.",+ "name": "The customer's name.",+ "email": "The customer's email address.",+ "company": "The company the customer belongs to.",+ "language": "The customer's preferred language code.",+ "tags": "The tags applied to the customer.",+ "custom_attributes": "Custom attributes stored on the customer.",+ "subscribed": "Whether the customer is subscribed to receive surveys.",+ "created": "When the customer was created.",+ "modified": "When the customer was last modified.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/settings.pyindex 55fdf47ebef1..4d2e94973740 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/settings.py@@ -1,9 +1,9 @@from dataclasses import dataclass, field-from products.warehouse_sources.backend.types import IncrementalField+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType-@dataclass+@dataclass(frozen=True)class SimplesatEndpointConfig:name: strpath: str@@ -14,13 +14,18 @@ class SimplesatEndpointConfig:method: str = "GET"# Simplesat object IDs are unique per account, so `id` is a safe primary key everywhere.primary_keys: list[str] = field(default_factory=lambda: ["id"])+ # Most list endpoints cap a page at 100 records; the largest page minimises round trips.+ page_size: int = 100+ # Maps an incremental field to the query param that filters on it server-side.+ incremental_param_by_field: dict[str, str] = field(default_factory=dict)+ partition_key: str | None = None-# Simplesat v1 top-level list endpoints. All are full-refresh only: while `answers` and-# `responses` accept optional start_date/end_date filters, the API's ordering guarantees for-# those filters aren't documented well enough to advance an incremental cursor safely, so a-# client-side scan would cost the same as a full refresh (see the implementing-warehouse-sources-# skill).+# Simplesat v1 top-level list endpoints. Only `customers` syncs incrementally, through its+# documented `created_after` / `modified_after` filters. `answers` and `responses` accept optional+# start_date/end_date filters, but the API's ordering guarantees for those filters aren't documented+# well enough to advance an incremental cursor safely, so they stay full refresh (see the+# implementing-warehouse-sources skill).SIMPLESAT_ENDPOINTS: dict[str, SimplesatEndpointConfig] = {"surveys": SimplesatEndpointConfig(name="surveys", path="/surveys", list_key="surveys", method="GET"),"questions": SimplesatEndpointConfig(name="questions", path="/questions", list_key="questions", method="GET"),@@ -28,8 +33,31 @@ SIMPLESAT_ENDPOINTS: dict[str, SimplesatEndpointConfig] = {"responses": SimplesatEndpointConfig(name="responses", path="/responses/search", list_key="responses", method="POST"),+ "customers": SimplesatEndpointConfig(+ name="customers",+ path="/customers",+ list_key="customers",+ method="GET",+ page_size=250,+ incremental_param_by_field={"modified": "modified_after", "created": "created_after"},+ partition_key="created",+ ),}ENDPOINTS = tuple(SIMPLESAT_ENDPOINTS.keys())-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}++def _datetime_field(name: str) -> IncrementalField:+ return {+ "label": name,+ "type": IncrementalFieldType.DateTime,+ "field": name,+ "field_type": IncrementalFieldType.DateTime,+ }+++INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {+ name: [_datetime_field(f) for f in config.incremental_param_by_field]+ for name, config in SIMPLESAT_ENDPOINTS.items()+ if config.incremental_param_by_field+}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/simplesat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/simplesat.pyindex ec16e815adc7..dfa009470cef 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/simplesat.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/simplesat.py@@ -1,4 +1,5 @@import dataclasses+from datetime import UTC, date, datetimefrom typing import Any, Optionalfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session@@ -17,8 +18,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.simplesat.SIMPLESAT_BASE_URL = "https://api.simplesat.io/api/v1"SIMPLESAT_HOST = "api.simplesat.io"-# The list endpoints return up to 100 records per page; the largest page minimises round trips.-PAGE_SIZE = 100# Cheap endpoint used to confirm an API key is genuine. The key is account-wide, so one probe# validates access to every list endpoint.DEFAULT_PROBE_PATH = "/surveys"@@ -38,21 +37,39 @@ def _headers() -> dict[str, str]:return {"Accept": "application/json"}+def _format_incremental_value(value: Any) -> str:+ if isinstance(value, datetime):+ utc_value = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)+ return utc_value.isoformat().replace("+00:00", "Z")+ if isinstance(value, date):+ return f"{value.isoformat()}T00:00:00Z"+ return str(value)++def simplesat_source(api_key: str,endpoint: str,team_id: int,job_id: str,resumable_source_manager: ResumableSourceManager[SimplesatResumeConfig],+ should_use_incremental_field: bool = False,+ db_incremental_field_last_value: Optional[Any] = None,+ incremental_field: Optional[str] = None,) -> SourceResponse:config = SIMPLESAT_ENDPOINTS[endpoint]is_post = config.method == "POST"+ params: dict[str, Any] = {"page_size": config.page_size}+ incremental_param = config.incremental_param_by_field.get(incremental_field or "")+ # The first incremental sync has no watermark yet, so it reads everything unfiltered.+ if should_use_incremental_field and incremental_param and db_incremental_field_last_value is not None:+ params[incremental_param] = _format_incremental_value(db_incremental_field_last_value)+endpoint_config: Endpoint = {"path": config.path,"method": "POST" if is_post else "GET",- "params": {"page_size": PAGE_SIZE},+ "params": params,# Simplesat wraps the page under a key named after the resource# (e.g. {"surveys": [...], "next": ...})."data_selector": config.list_key,@@ -114,6 +131,9 @@ def simplesat_source(primary_keys=config.primary_keys,partition_count=1,partition_size=1,+ partition_mode="datetime" if config.partition_key else None,+ partition_format="month" if config.partition_key else None,+ partition_keys=[config.partition_key] if config.partition_key else None,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/source.pyindex 301a25dcb878..959a01190e33 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/source.py@@ -58,7 +58,7 @@ class SimplesatSource(ResumableSource[SimplesatSourceConfig, SimplesatResumeConfreleaseStatus=ReleaseStatus.BETA,caption="""Enter your Simplesat API key to pull your customer satisfaction survey data into the PostHog Data warehouse.-You can create an API key under **Settings → API keys** in [Simplesat](https://app.simplesat.io/). The key must include read scopes for each resource you want to sync — surveys, questions, answers, and responses.+You can create an API key under **Settings → API keys** in [Simplesat](https://app.simplesat.io/). The key must include read scopes for each resource you want to sync — surveys, questions, answers, responses, and customers.""",iconPath="/static/services/simplesat.png",docsUrl="https://posthog.com/docs/cdp/sources/simplesat",@@ -99,9 +99,8 @@ You can create an API key under **Settings → API keys** in [Simplesat](https:/force_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- # Every endpoint is full refresh only — Simplesat's list endpoints expose no reliably- # ordered server-side timestamp filter, so there is no incremental cursor to advance- # (INCREMENTAL_FIELDS is empty, so build_endpoint_schemas marks each as full refresh).+ # Only endpoints with a server-side timestamp filter (customers) have INCREMENTAL_FIELDS;+ # build_endpoint_schemas marks the rest as full refresh.return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)def validate_credentials(@@ -133,4 +132,7 @@ You can create an API key under **Settings → API keys** in [Simplesat](https:/team_id=inputs.team_id,job_id=inputs.job_id,resumable_source_manager=resumable_source_manager,+ should_use_incremental_field=inputs.should_use_incremental_field,+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,+ incremental_field=inputs.incremental_field,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/tests/test_simplesat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/tests/test_simplesat.pyindex 13f9acf8f305..356b7b19ff42 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/tests/test_simplesat.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/tests/test_simplesat.py@@ -1,4 +1,6 @@+from datetime import UTC, date, datetimefrom typing import Any, Optional+from urllib.parse import parse_qs, urlparseimport requestsfrom parameterized import parameterized@@ -60,6 +62,64 @@ class TestPagination:assert result.urls == [next_url]+class TestCustomersRequestShaping:+ @parameterized.expand(+ [+ ("full_refresh", None, None, {}),+ ("first_incremental_run_has_no_filter", "modified", None, {}),+ (+ "modified_aware_datetime",+ "modified",+ datetime(2026, 3, 1, 12, 30, 45, 123456, tzinfo=UTC),+ {"modified_after": "2026-03-01T12:30:45.123456Z"},+ ),+ (+ "modified_naive_datetime_is_utc",+ "modified",+ datetime(2026, 3, 1, 12, 30, 45),+ {"modified_after": "2026-03-01T12:30:45Z"},+ ),+ ("created_date", "created", date(2026, 3, 1), {"created_after": "2026-03-01T00:00:00Z"}),+ ("created_string", "created", "2026-03-01T00:00:00Z", {"created_after": "2026-03-01T00:00:00Z"}),+ ]+ )+ def test_incremental_filter_params(+ self,+ _name: str,+ incremental_field: Optional[str],+ last_value: Any,+ expected_filter: dict[str, str],+ ) -> None:+ page_2 = f"{SIMPLESAT_BASE_URL}/customers?page=2&page_size=250"+ result = _driver().run(+ "customers",+ [+ _response("customers", [{"id": 1}], next_url=page_2),+ _response("customers", [{"id": 2}], next_url=None),+ ],+ incremental_field=incremental_field,+ db_incremental_field_last_value=last_value,+ )++ assert result.raised is None+ assert result.rows == [{"id": 1}, {"id": 2}]+ first_query = parse_qs(urlparse(result.urls[0]).query)+ assert first_query == {"page_size": ["250"], **{k: [v] for k, v in expected_filter.items()}}+ # Later pages follow the `next` URL verbatim.+ assert result.urls[1] == page_2++ @parameterized.expand([(e,) for e in ENDPOINTS if e != "customers"])+ def test_full_refresh_endpoints_ignore_incremental_inputs(self, endpoint: str) -> None:+ result = _driver().run(+ endpoint,+ always(_response(endpoint, [])),+ incremental_field="modified",+ db_incremental_field_last_value=datetime(2026, 3, 1, tzinfo=UTC),+ )++ assert parse_qs(urlparse(result.urls[0]).query) == {"page_size": ["100"]}++class TestHostPinning:def test_off_origin_next_url_is_rejected_before_it_is_fetched(self) -> None:evil = "https://evil.example.com/api/v1/surveys?page=2"@@ -147,15 +207,16 @@ class TestValidateCredentials:class TestSimplesatSourceResponse:- @parameterized.expand([(e,) for e in ENDPOINTS])- def test_source_response_shape(self, endpoint: str) -> None:+ @parameterized.expand([(e, ["created"] if e == "customers" else None) for e in ENDPOINTS])+ def test_source_response_shape(self, endpoint: str, expected_partition_keys: Optional[list[str]]) -> None:response = _driver().run(endpoint, always(_response(endpoint, []))).responseassert response is not Noneassert response.name == endpointassert response.primary_keys == ["id"]- # No stable creation timestamp is guaranteed across every object, so we don't partition.- assert response.partition_mode is None+ # Only customers documents a stable creation timestamp to partition on.+ assert response.partition_keys == expected_partition_keys+ assert response.partition_mode == ("datetime" if expected_partition_keys else None)def test_every_endpoint_uses_id_primary_key(self) -> None:assert all(config.primary_keys == ["id"] for config in SIMPLESAT_ENDPOINTS.values())