Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 1 missing table to the Simplesat source (#120553)

Tom Owersf4a789c4fc75
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 5291efa32790..1677e2dbaffc 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -1029,7 +1029,6 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/similarweb/similarweb.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/simplecast/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/simplecast/simplecast.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/simplesat.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/skyvern/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/skyvern/skyvern.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index eccbbcff6374..a13a6718c9e4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -11125,7 +11125,7 @@ Today (4): `answers`, `questions`, `responses`, `surveys`
Diffed against: <https://developer.simplesat.io/api/Simplesat%20API%20(v1)%20OpenAPI.yaml>
-- [ ] `customers (GET /api/v1/customers)` — lookup table resolving the customer referenced on every synced response, so CSAT/NPS can be sliced by account (high)
+- [x] `customers (GET /api/v1/customers)` — lookup table resolving the customer referenced on every synced response, so CSAT/NPS can be sliced by account (high)
Note: The v1 OpenAPI defines only six resources: answers, customers, team-members, responses, surveys, questions. PostHog syncs four of them, so coverage is proportionate — only customers is a real miss. team-members exposes GET /api/v1/team-members/{team_member_id} but no list endpoint (the collection path is POST-only), so it cannot be synced as a table without hydrating ids from responses; I did not list it as a gap for that reason.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/canonical_descriptions.py
index de9e965860ac..c6b5a83c5829 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/canonical_descriptions.py
@@ -63,4 +63,21 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"modified": "When the response was last modified.",
},
},
+ "customers": {
+ "description": "A customer (survey respondent) in the Simplesat account.",
+ "docs_url": "https://developer.simplesat.io/api",
+ "columns": {
+ "id": "The unique ID of the customer.",
+ "external_id": "The customer's ID in an external system, such as a help desk or CRM.",
+ "name": "The customer's name.",
+ "email": "The customer's email address.",
+ "company": "The company the customer belongs to.",
+ "language": "The customer's preferred language code.",
+ "tags": "The tags applied to the customer.",
+ "custom_attributes": "Custom attributes stored on the customer.",
+ "subscribed": "Whether the customer is subscribed to receive surveys.",
+ "created": "When the customer was created.",
+ "modified": "When the customer was last modified.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/settings.py
index 55fdf47ebef1..4d2e94973740 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/settings.py
@@ -1,9 +1,9 @@
from dataclasses import dataclass, field
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
-@dataclass
+@dataclass(frozen=True)
class SimplesatEndpointConfig:
name: str
path: str
@@ -14,13 +14,18 @@ class SimplesatEndpointConfig:
method: str = "GET"
# Simplesat object IDs are unique per account, so `id` is a safe primary key everywhere.
primary_keys: list[str] = field(default_factory=lambda: ["id"])
+ # Most list endpoints cap a page at 100 records; the largest page minimises round trips.
+ page_size: int = 100
+ # Maps an incremental field to the query param that filters on it server-side.
+ incremental_param_by_field: dict[str, str] = field(default_factory=dict)
+ partition_key: str | None = None
-# Simplesat v1 top-level list endpoints. All are full-refresh only: while `answers` and
-# `responses` accept optional start_date/end_date filters, the API's ordering guarantees for
-# those filters aren't documented well enough to advance an incremental cursor safely, so a
-# client-side scan would cost the same as a full refresh (see the implementing-warehouse-sources
-# skill).
+# Simplesat v1 top-level list endpoints. Only `customers` syncs incrementally, through its
+# documented `created_after` / `modified_after` filters. `answers` and `responses` accept optional
+# start_date/end_date filters, but the API's ordering guarantees for those filters aren't documented
+# well enough to advance an incremental cursor safely, so they stay full refresh (see the
+# implementing-warehouse-sources skill).
SIMPLESAT_ENDPOINTS: dict[str, SimplesatEndpointConfig] = {
"surveys": SimplesatEndpointConfig(name="surveys", path="/surveys", list_key="surveys", method="GET"),
"questions": SimplesatEndpointConfig(name="questions", path="/questions", list_key="questions", method="GET"),
@@ -28,8 +33,31 @@ SIMPLESAT_ENDPOINTS: dict[str, SimplesatEndpointConfig] = {
"responses": SimplesatEndpointConfig(
name="responses", path="/responses/search", list_key="responses", method="POST"
),
+ "customers": SimplesatEndpointConfig(
+ name="customers",
+ path="/customers",
+ list_key="customers",
+ method="GET",
+ page_size=250,
+ incremental_param_by_field={"modified": "modified_after", "created": "created_after"},
+ partition_key="created",
+ ),
}
ENDPOINTS = tuple(SIMPLESAT_ENDPOINTS.keys())
-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
+
+def _datetime_field(name: str) -> IncrementalField:
+ return {
+ "label": name,
+ "type": IncrementalFieldType.DateTime,
+ "field": name,
+ "field_type": IncrementalFieldType.DateTime,
+ }
+
+
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ name: [_datetime_field(f) for f in config.incremental_param_by_field]
+ for name, config in SIMPLESAT_ENDPOINTS.items()
+ if config.incremental_param_by_field
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/simplesat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/simplesat.py
index ec16e815adc7..dfa009470cef 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/simplesat.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/simplesat.py
@@ -1,4 +1,5 @@
import dataclasses
+from datetime import UTC, date, datetime
from typing import Any, Optional
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
@@ -17,8 +18,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.simplesat.
SIMPLESAT_BASE_URL = "https://api.simplesat.io/api/v1"
SIMPLESAT_HOST = "api.simplesat.io"
-# The list endpoints return up to 100 records per page; the largest page minimises round trips.
-PAGE_SIZE = 100
# Cheap endpoint used to confirm an API key is genuine. The key is account-wide, so one probe
# validates access to every list endpoint.
DEFAULT_PROBE_PATH = "/surveys"
@@ -38,21 +37,39 @@ def _headers() -> dict[str, str]:
return {"Accept": "application/json"}
+def _format_incremental_value(value: Any) -> str:
+ if isinstance(value, datetime):
+ utc_value = value.replace(tzinfo=UTC) if value.tzinfo is None else value.astimezone(UTC)
+ return utc_value.isoformat().replace("+00:00", "Z")
+ if isinstance(value, date):
+ return f"{value.isoformat()}T00:00:00Z"
+ return str(value)
+
+
def simplesat_source(
api_key: str,
endpoint: str,
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[SimplesatResumeConfig],
+ should_use_incremental_field: bool = False,
+ db_incremental_field_last_value: Optional[Any] = None,
+ incremental_field: Optional[str] = None,
) -> SourceResponse:
config = SIMPLESAT_ENDPOINTS[endpoint]
is_post = config.method == "POST"
+ params: dict[str, Any] = {"page_size": config.page_size}
+ incremental_param = config.incremental_param_by_field.get(incremental_field or "")
+ # The first incremental sync has no watermark yet, so it reads everything unfiltered.
+ if should_use_incremental_field and incremental_param and db_incremental_field_last_value is not None:
+ params[incremental_param] = _format_incremental_value(db_incremental_field_last_value)
+
endpoint_config: Endpoint = {
"path": config.path,
"method": "POST" if is_post else "GET",
- "params": {"page_size": PAGE_SIZE},
+ "params": params,
# Simplesat wraps the page under a key named after the resource
# (e.g. {"surveys": [...], "next": ...}).
"data_selector": config.list_key,
@@ -114,6 +131,9 @@ def simplesat_source(
primary_keys=config.primary_keys,
partition_count=1,
partition_size=1,
+ partition_mode="datetime" if config.partition_key else None,
+ partition_format="month" if config.partition_key else None,
+ partition_keys=[config.partition_key] if config.partition_key else None,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/source.py
index 301a25dcb878..959a01190e33 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/source.py
@@ -58,7 +58,7 @@ class SimplesatSource(ResumableSource[SimplesatSourceConfig, SimplesatResumeConf
releaseStatus=ReleaseStatus.BETA,
caption="""Enter your Simplesat API key to pull your customer satisfaction survey data into the PostHog Data warehouse.
-You can create an API key under **Settings → API keys** in [Simplesat](https://app.simplesat.io/). The key must include read scopes for each resource you want to sync — surveys, questions, answers, and responses.
+You can create an API key under **Settings → API keys** in [Simplesat](https://app.simplesat.io/). The key must include read scopes for each resource you want to sync — surveys, questions, answers, responses, and customers.
""",
iconPath="/static/services/simplesat.png",
docsUrl="https://posthog.com/docs/cdp/sources/simplesat",
@@ -99,9 +99,8 @@ You can create an API key under **Settings → API keys** in [Simplesat](https:/
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- # Every endpoint is full refresh only — Simplesat's list endpoints expose no reliably
- # ordered server-side timestamp filter, so there is no incremental cursor to advance
- # (INCREMENTAL_FIELDS is empty, so build_endpoint_schemas marks each as full refresh).
+ # Only endpoints with a server-side timestamp filter (customers) have INCREMENTAL_FIELDS;
+ # build_endpoint_schemas marks the rest as full refresh.
return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
def validate_credentials(
@@ -133,4 +132,7 @@ You can create an API key under **Settings → API keys** in [Simplesat](https:/
team_id=inputs.team_id,
job_id=inputs.job_id,
resumable_source_manager=resumable_source_manager,
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=inputs.db_incremental_field_last_value,
+ incremental_field=inputs.incremental_field,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/tests/test_simplesat.py b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/tests/test_simplesat.py
index 13f9acf8f305..356b7b19ff42 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/tests/test_simplesat.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/simplesat/tests/test_simplesat.py
@@ -1,4 +1,6 @@
+from datetime import UTC, date, datetime
from typing import Any, Optional
+from urllib.parse import parse_qs, urlparse
import requests
from parameterized import parameterized
@@ -60,6 +62,64 @@ class TestPagination:
assert result.urls == [next_url]
+class TestCustomersRequestShaping:
+ @parameterized.expand(
+ [
+ ("full_refresh", None, None, {}),
+ ("first_incremental_run_has_no_filter", "modified", None, {}),
+ (
+ "modified_aware_datetime",
+ "modified",
+ datetime(2026, 3, 1, 12, 30, 45, 123456, tzinfo=UTC),
+ {"modified_after": "2026-03-01T12:30:45.123456Z"},
+ ),
+ (
+ "modified_naive_datetime_is_utc",
+ "modified",
+ datetime(2026, 3, 1, 12, 30, 45),
+ {"modified_after": "2026-03-01T12:30:45Z"},
+ ),
+ ("created_date", "created", date(2026, 3, 1), {"created_after": "2026-03-01T00:00:00Z"}),
+ ("created_string", "created", "2026-03-01T00:00:00Z", {"created_after": "2026-03-01T00:00:00Z"}),
+ ]
+ )
+ def test_incremental_filter_params(
+ self,
+ _name: str,
+ incremental_field: Optional[str],
+ last_value: Any,
+ expected_filter: dict[str, str],
+ ) -> None:
+ page_2 = f"{SIMPLESAT_BASE_URL}/customers?page=2&page_size=250"
+ result = _driver().run(
+ "customers",
+ [
+ _response("customers", [{"id": 1}], next_url=page_2),
+ _response("customers", [{"id": 2}], next_url=None),
+ ],
+ incremental_field=incremental_field,
+ db_incremental_field_last_value=last_value,
+ )
+
+ assert result.raised is None
+ assert result.rows == [{"id": 1}, {"id": 2}]
+ first_query = parse_qs(urlparse(result.urls[0]).query)
+ assert first_query == {"page_size": ["250"], **{k: [v] for k, v in expected_filter.items()}}
+ # Later pages follow the `next` URL verbatim.
+ assert result.urls[1] == page_2
+
+ @parameterized.expand([(e,) for e in ENDPOINTS if e != "customers"])
+ def test_full_refresh_endpoints_ignore_incremental_inputs(self, endpoint: str) -> None:
+ result = _driver().run(
+ endpoint,
+ always(_response(endpoint, [])),
+ incremental_field="modified",
+ db_incremental_field_last_value=datetime(2026, 3, 1, tzinfo=UTC),
+ )
+
+ assert parse_qs(urlparse(result.urls[0]).query) == {"page_size": ["100"]}
+
+
class TestHostPinning:
def test_off_origin_next_url_is_rejected_before_it_is_fetched(self) -> None:
evil = "https://evil.example.com/api/v1/surveys?page=2"
@@ -147,15 +207,16 @@ class TestValidateCredentials:
class TestSimplesatSourceResponse:
- @parameterized.expand([(e,) for e in ENDPOINTS])
- def test_source_response_shape(self, endpoint: str) -> None:
+ @parameterized.expand([(e, ["created"] if e == "customers" else None) for e in ENDPOINTS])
+ def test_source_response_shape(self, endpoint: str, expected_partition_keys: Optional[list[str]]) -> None:
response = _driver().run(endpoint, always(_response(endpoint, []))).response
assert response is not None
assert response.name == endpoint
assert response.primary_keys == ["id"]
- # No stable creation timestamp is guaranteed across every object, so we don't partition.
- assert response.partition_mode is None
+ # Only customers documents a stable creation timestamp to partition on.
+ assert response.partition_keys == expected_partition_keys
+ assert response.partition_mode == ("datetime" if expected_partition_keys else None)
def test_every_endpoint_uses_id_primary_key(self) -> None:
assert all(config.primary_keys == ["id"] for config in SIMPLESAT_ENDPOINTS.values())