Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): add 1 missing table to the Qualaroo source (#120840)

Tom Owers3e0cfb97dc47
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
index a80e3be9c6ec..9370c56295eb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md
@@ -9924,7 +9924,7 @@ Today (1): `nudges`
Diffed against: <https://help.qualaroo.com/hc/en-us/articles/201969438-The-REST-Reporting-API>
-- [ ] `GET /nudges/{nudge_id}/responses.json` — the survey answers themselves - the only analytical data the Reporting API exposes; today the source syncs survey definitions with no responses (high)
+- [x] `GET /nudges/{nudge_id}/responses.json` — the survey answers themselves - the only analytical data the Reporting API exposes; today the source syncs survey definitions with no responses (high)
Note: The REST Reporting API is essentially two endpoints (nudges list, and per-nudge responses with start_date/end_date/offset/limit paging). The source's settings define only the nudges endpoint - no dynamic table discovery - so the entire response dataset is unavailable. Adding it requires a fan-out over nudge IDs, same shape as other per-parent sources.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/canonical_descriptions.py
index 80c10c0fc7c2..87cc2ed23184 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/canonical_descriptions.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/canonical_descriptions.py
@@ -19,4 +19,21 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"updated_at": "When the nudge was last updated.",
},
},
+ "responses": {
+ "description": "One respondent's response to a Qualaroo nudge (survey), with their answers and metadata.",
+ "docs_url": "https://help.qualaroo.com/hc/en-us/articles/201969438-The-REST-Reporting-API",
+ "columns": {
+ "id": "The ID of the response record.",
+ "nudge_id": "The ID of the nudge (survey) that the response belongs to.",
+ "time": "When the response was recorded, in the account time zone with its UTC offset.",
+ "token": "An anonymous identifier of the respondent.",
+ "ip_address": "The IP address of the respondent.",
+ "identity": "The identity that you tagged the respondent with.",
+ "page": "The page of your website where the respondent interacted with the survey.",
+ "referrer": "The URL of the referrer page, if there was one.",
+ "user_agent": "The User-Agent of the respondent's browser.",
+ "answered_questions": "A map of the questions that the respondent answered, with each question ID, title, answer text, selected option ID and comment.",
+ "properties": "The custom name/value properties that you tagged the respondent with.",
+ },
+ },
}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/qualaroo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/qualaroo.py
index 6c2401c72efe..32387338a40c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/qualaroo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/qualaroo.py
@@ -1,34 +1,137 @@
import dataclasses
+from datetime import UTC, datetime, timedelta
from typing import Any, Optional
from requests.auth import HTTPBasicAuth
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
RESTAPIConfig,
rest_api_resource,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ build_dependent_resource,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
OffsetPaginator,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+ ClientConfig,
+ IncrementalConfig,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.qualaroo.settings import QUALAROO_ENDPOINTS
+from products.warehouse_sources.backend.temporal.data_imports.sources.qualaroo.settings import (
+ PAGE_SIZE,
+ QUALAROO_ENDPOINTS,
+)
QUALAROO_BASE_URL = "https://api.qualaroo.com/api/v1"
-# The Reporting API caps a page at 500 records; the largest page minimises round trips.
-PAGE_SIZE = 500
# Cheap endpoint used to confirm the credentials are genuine. The API key/secret pair is
# account-wide, so one probe validates access to every list endpoint.
DEFAULT_PROBE_PATH = "/nudges.json"
[email protected](frozen=False)
class QualarooResumeConfig:
# Offset of the next page to fetch. Offset pagination is deterministic, so a crashed
# full-refresh sync resumes from the page after the last one yielded; merge dedupes on `id`.
offset: int = 0
+ # Progress of an endpoint that is listed once per nudge: the nudges that are done, the nudge in
+ # progress, and the page cursor of that nudge. The shared fan-out helper owns the shape.
+ fanout_state: Optional[dict[str, Any]] = None
+ # `end_date` fixed at sync start, so responses that arrive mid fan-out are left for the next run.
+ sync_end: Optional[str] = None
+
+
+def _client_config(api_key: str, api_secret: str) -> ClientConfig:
+ return {
+ "base_url": QUALAROO_BASE_URL,
+ "headers": {"Accept": "application/json"},
+ # Qualaroo uses HTTP Basic auth with the API key as username and the secret as password.
+ "auth": {"type": "http_basic", "username": api_key, "password": api_secret},
+ # Qualaroo has no top-level `total`; termination is a short/empty page (OffsetPaginator default).
+ "paginator": OffsetPaginator(limit=PAGE_SIZE, total_path=None),
+ }
+
+
+def _to_unix_seconds(value: Any) -> Optional[int]:
+ parsed = parse_datetime_value(value)
+ if parsed is None:
+ # No watermark yet: omit `start_date`, so the API starts from the first response.
+ return None
+ # The docs do not say whether `start_date` is inclusive. A one-second overlap keeps rows that
+ # share the watermark second, and merge dedupes the rows it reads again.
+ return int((parsed - timedelta(seconds=1)).timestamp())
+
+
+def _incremental_window(cursor_path: str, sync_end: str) -> IncrementalConfig:
+ return {
+ "cursor_path": cursor_path,
+ "start_param": "start_date",
+ "end_param": "end_date",
+ "end_value": sync_end,
+ "convert": _to_unix_seconds,
+ }
+
+
+def _fanout_source(
+ api_key: str,
+ api_secret: str,
+ endpoint: str,
+ team_id: int,
+ job_id: str,
+ resumable_source_manager: ResumableSourceManager[QualarooResumeConfig],
+ should_use_incremental_field: bool,
+ db_incremental_field_last_value: Optional[Any],
+ incremental_field: Optional[str],
+) -> SourceResponse:
+ config = QUALAROO_ENDPOINTS[endpoint]
+ assert config.fanout is not None
+
+ initial_fanout_state: Optional[dict[str, Any]] = None
+ sync_end: Optional[str] = None
+ if resumable_source_manager.can_resume():
+ resume = resumable_source_manager.load_state()
+ if resume is not None:
+ initial_fanout_state = resume.fanout_state
+ sync_end = resume.sync_end
+ run_sync_end = sync_end or datetime.now(UTC).isoformat()
+
+ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:
+ if state is not None:
+ resumable_source_manager.save_state(QualarooResumeConfig(fanout_state=state, sync_end=run_sync_end))
+
+ resource = build_dependent_resource(
+ endpoint_configs=QUALAROO_ENDPOINTS,
+ child_endpoint=endpoint,
+ fanout=config.fanout,
+ # Survey answers are free text that the sample scrubbers cannot reliably redact.
+ client_config={**_client_config(api_key, api_secret), "capture": False},
+ path_format_values={},
+ team_id=team_id,
+ job_id=job_id,
+ db_incremental_field_last_value=db_incremental_field_last_value,
+ should_use_incremental_field=should_use_incremental_field,
+ incremental_field=incremental_field,
+ incremental_config_factory=lambda cursor_path: _incremental_window(cursor_path, run_sync_end),
+ # The client OffsetPaginator already sends `limit` on every request.
+ page_size_param=None,
+ parent_endpoint_extra={"data_selector_malformed_retryable": True},
+ child_endpoint_extra={"data_selector_malformed_retryable": True},
+ resume_hook=save_fanout_checkpoint,
+ initial_paginator_state=initial_fanout_state,
+ )
+
+ return SourceResponse(
+ name=endpoint,
+ items=lambda: resource,
+ primary_keys=config.primary_keys,
+ partition_count=1,
+ partition_size=1,
+ )
def qualaroo_source(
@@ -38,19 +141,26 @@ def qualaroo_source(
team_id: int,
job_id: str,
resumable_source_manager: ResumableSourceManager[QualarooResumeConfig],
+ should_use_incremental_field: bool = False,
db_incremental_field_last_value: Optional[Any] = None,
+ incremental_field: Optional[str] = None,
) -> SourceResponse:
config = QUALAROO_ENDPOINTS[endpoint]
+ if config.fanout is not None:
+ return _fanout_source(
+ api_key,
+ api_secret,
+ endpoint,
+ team_id,
+ job_id,
+ resumable_source_manager,
+ should_use_incremental_field,
+ db_incremental_field_last_value,
+ incremental_field,
+ )
rest_config: RESTAPIConfig = {
- "client": {
- "base_url": QUALAROO_BASE_URL,
- "headers": {"Accept": "application/json"},
- # Qualaroo uses HTTP Basic auth with the API key as username and the secret as password.
- "auth": {"type": "http_basic", "username": api_key, "password": api_secret},
- # Qualaroo has no top-level `total`; termination is a short/empty page (OffsetPaginator default).
- "paginator": OffsetPaginator(limit=PAGE_SIZE, total_path=None),
- },
+ "client": _client_config(api_key, api_secret),
# Per-resource settings are fully specified below, so no shared defaults are needed.
"resource_defaults": {},
"resources": [
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/settings.py
index ff75f8cdbda5..e55164398776 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/settings.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/settings.py
@@ -1,27 +1,66 @@
from dataclasses import dataclass, field
+from typing import Optional
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ DependentEndpointConfig,
+)
+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+# The Reporting API caps a page at 500 records; the largest page minimises round trips.
+PAGE_SIZE = 500
-@dataclass
+
+@dataclass(frozen=False)
class QualarooEndpointConfig:
name: str
path: str
# Qualaroo nudge (survey) IDs are unique within an account, so `id` is a safe primary key.
primary_keys: list[str] = field(default_factory=lambda: ["id"])
+ page_size: int = PAGE_SIZE
+ incremental_fields: list[IncrementalField] = field(default_factory=list)
+ default_incremental_field: Optional[str] = None
+ fanout: Optional[DependentEndpointConfig] = None
-# Qualaroo REST Reporting API top-level list endpoints. Only account-wide lists are exposed here:
-# per-survey responses live at /nudges/{id}/responses.json and require a parent nudge ID, so they
-# are a fan-out stream and are intentionally skipped for now.
+# Qualaroo REST Reporting API endpoints. `nudges` is the account-wide survey list. `responses` is
+# listed once per nudge at /nudges/{nudge_id}/responses.json.
#
-# All endpoints are full-refresh only. The responses endpoint supports server-side start_date/
-# end_date filters, but the nudges list has no reliably ordered timestamp filter, so there is no
-# incremental cursor to advance (see the implementing-warehouse-sources skill).
+# The nudges list has no server-side timestamp filter, so it is full refresh only. Responses take
+# a `start_date` filter in UNIX seconds against the submission `time`, so they sync incrementally.
QUALAROO_ENDPOINTS: dict[str, QualarooEndpointConfig] = {
"nudges": QualarooEndpointConfig(name="nudges", path="/nudges.json"),
+ # The docs do not say that a response id is unique across nudges, so the nudge id is part of the key.
+ "responses": QualarooEndpointConfig(
+ name="responses",
+ path="/nudges/{nudge_id}/responses.json",
+ primary_keys=["nudge_id", "id"],
+ incremental_fields=[
+ {
+ "label": "time",
+ "type": IncrementalFieldType.DateTime,
+ "field": "time",
+ "field_type": IncrementalFieldType.DateTime,
+ }
+ ],
+ default_incremental_field="time",
+ fanout=DependentEndpointConfig(
+ parent_name="nudges",
+ resolve_param="nudge_id",
+ resolve_field="id",
+ include_from_parent=["id"],
+ parent_field_renames={"id": "nudge_id"},
+ # Without `order=asc` the API returns newest first, so a response submitted mid-sync
+ # shifts the offset pages and one row is read twice while another is skipped.
+ child_params={"order": "asc"},
+ # The API answers 404 for a nudge deleted between the nudge listing and this request.
+ # That nudge has no responses to sync, so it must not fail the nudges that remain.
+ child_response_actions=[{"status_code": 404, "action": "ignore"}],
+ ),
+ ),
}
ENDPOINTS = tuple(QUALAROO_ENDPOINTS.keys())
-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}
+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {
+ name: config.incremental_fields for name, config in QUALAROO_ENDPOINTS.items() if config.incremental_fields
+}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/source.py
index 6f553e927299..a7bafabf0358 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/source.py
@@ -107,10 +107,9 @@ You can find your API key and secret under **Settings → API** in [Qualaroo](ht
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
- # Every endpoint is full refresh only — the nudges list exposes no reliably ordered
- # server-side timestamp filter, so INCREMENTAL_FIELDS is empty and every schema is
- # non-incremental / non-append.
- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
+ # Responses are merge only: `start_date` re-reads the watermark second, which append would
+ # store twice.
+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=("responses",))
def validate_credentials(
self,
@@ -141,5 +140,9 @@ You can find your API key and secret under **Settings → API** in [Qualaroo](ht
team_id=inputs.team_id,
job_id=inputs.job_id,
resumable_source_manager=resumable_source_manager,
- db_incremental_field_last_value=None, # every Qualaroo endpoint is full refresh
+ should_use_incremental_field=inputs.should_use_incremental_field,
+ db_incremental_field_last_value=(
+ inputs.db_incremental_field_last_value if inputs.should_use_incremental_field else None
+ ),
+ incremental_field=inputs.incremental_field,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/tests/test_qualaroo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/tests/test_qualaroo.py
index 5707abd2eb69..722377e3124e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/tests/test_qualaroo.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/tests/test_qualaroo.py
@@ -1,3 +1,4 @@
+from datetime import UTC, datetime
from typing import Any
import requests
@@ -12,6 +13,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
ScriptedResponse,
SourceDriver,
always,
+ route,
scripted_network,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.qualaroo import (
@@ -113,10 +115,129 @@ class TestSourceResponse:
assert result.response is not None
assert result.response.name == endpoint
- assert result.response.primary_keys == ["id"]
+ assert result.response.primary_keys == QUALAROO_ENDPOINTS[endpoint].primary_keys
# No stable creation timestamp is guaranteed across every object, so we don't partition.
assert result.response.partition_mode is None
- def test_every_endpoint_uses_id_primary_key(self) -> None:
- assert all(config.primary_keys == ["id"] for config in QUALAROO_ENDPOINTS.values())
- assert set(QUALAROO_ENDPOINTS) == set(ENDPOINTS)
+
+class TestResponsesFanOut:
+ def test_walks_every_nudge_and_every_response_page(self) -> None:
+ result = _driver().run(
+ "responses",
+ route(
+ {
+ "/nudges.json": [ScriptedResponse(json=[{"id": 11}, {"id": 22}, {"id": 33}])],
+ "/nudges/11/responses.json": [
+ ScriptedResponse(
+ json=[{"id": i, "time": "2024-03-01 10:00:00 -0700"} for i in range(PAGE_SIZE)]
+ ),
+ ScriptedResponse(json=[{"id": 900, "answered_questions": {}, "properties": None}]),
+ ],
+ "/nudges/22/responses.json": [ScriptedResponse(status=404, json={"error": "Not Found"})],
+ "/nudges/33/responses.json": [ScriptedResponse(json=[{"id": 1}])],
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert len(result.rows) == PAGE_SIZE + 2
+ assert result.rows[PAGE_SIZE] == {"id": 900, "answered_questions": {}, "properties": None, "nudge_id": 11}
+ # Response ids are keyed with their nudge, so the same id under two nudges stays two rows.
+ assert {(row["nudge_id"], row["id"]) for row in result.rows if row["id"] == 1} == {(11, 1), (33, 1)}
+ assert [(request.path, request.param("offset")) for request in result.requests] == [
+ ("/api/v1/nudges.json", "0"),
+ ("/api/v1/nudges/11/responses.json", "0"),
+ ("/api/v1/nudges/11/responses.json", str(PAGE_SIZE)),
+ ("/api/v1/nudges/22/responses.json", "0"),
+ ("/api/v1/nudges/33/responses.json", "0"),
+ ]
+ child_requests = [request for request in result.requests if request.path.endswith("/responses.json")]
+ assert {request.param("order") for request in child_requests} == {"asc"}
+ assert {request.param("limit") for request in result.requests} == {str(PAGE_SIZE)}
+
+ @parameterized.expand(
+ [
+ ("full_refresh", {"should_use_incremental_field": False}, None),
+ ("first_incremental_run", {"incremental_field": "time"}, None),
+ (
+ "watermark_string_with_offset",
+ {"incremental_field": "time", "db_incremental_field_last_value": "2024-03-01 10:00:00 -0700"},
+ str(int(datetime(2024, 3, 1, 17, 0, 0, tzinfo=UTC).timestamp()) - 1),
+ ),
+ (
+ "watermark_naive_datetime_is_utc",
+ {"incremental_field": "time", "db_incremental_field_last_value": datetime(2024, 3, 1, 17, 0, 0)},
+ str(int(datetime(2024, 3, 1, 17, 0, 0, tzinfo=UTC).timestamp()) - 1),
+ ),
+ ]
+ )
+ def test_start_date_follows_the_watermark(
+ self, _name: str, inputs: dict[str, Any], expected_start_date: str | None
+ ) -> None:
+ result = _driver().run(
+ "responses",
+ route(
+ {
+ "/nudges.json": [ScriptedResponse(json=[{"id": 11}])],
+ "/nudges/11/responses.json": [ScriptedResponse(json=[{"id": 1}])],
+ }
+ ),
+ **inputs,
+ )
+
+ assert result.raised is None
+ assert [request.param("start_date") for request in result.requests] == [None, expected_start_date]
+ end_dates = [request.param("end_date") for request in result.requests]
+ assert end_dates[0] is None
+ assert (end_dates[1] is not None) == inputs.get("should_use_incremental_field", True)
+
+ def test_resumed_incremental_sync_keeps_the_original_end_date(self) -> None:
+ sync_end = "2024-03-02T00:00:00+00:00"
+ result = _driver().run(
+ "responses",
+ route(
+ {
+ "/nudges.json": [ScriptedResponse(json=[{"id": 11}, {"id": 22}])],
+ "/nudges/22/responses.json": [ScriptedResponse(json=[{"id": 7, "time": "2024-03-01 10:00:00"}])],
+ }
+ ),
+ resume_state=QualarooResumeConfig(
+ fanout_state={"completed": ["/nudges/11/responses.json"], "current": None, "child_state": None},
+ sync_end=sync_end,
+ ),
+ incremental_field="time",
+ db_incremental_field_last_value="2024-03-01 00:00:00",
+ )
+
+ assert result.raised is None
+ child_requests = [request for request in result.requests if request.path.endswith("/responses.json")]
+ expected_end_date = str(int(datetime(2024, 3, 2, tzinfo=UTC).timestamp()) - 1)
+ assert [request.param("end_date") for request in child_requests] == [expected_end_date]
+ assert all(state.sync_end == sync_end for state in result.saved_states)
+
+ def test_resumes_inside_the_nudge_in_progress(self) -> None:
+ result = _driver().run(
+ "responses",
+ route(
+ {
+ "/nudges.json": [ScriptedResponse(json=[{"id": 11}, {"id": 22}, {"id": 33}])],
+ "/nudges/22/responses.json": [ScriptedResponse(json=[{"id": 7}])],
+ "/nudges/33/responses.json": [ScriptedResponse(json=[])],
+ }
+ ),
+ resume_state=QualarooResumeConfig(
+ fanout_state={
+ "completed": ["/nudges/11/responses.json"],
+ "current": "/nudges/22/responses.json",
+ "child_state": {"offset": PAGE_SIZE},
+ }
+ ),
+ )
+
+ assert result.raised is None
+ assert result.rows == [{"id": 7, "nudge_id": 22}]
+ assert [(request.path, request.param("offset")) for request in result.requests] == [
+ ("/api/v1/nudges.json", "0"),
+ ("/api/v1/nudges/22/responses.json", str(PAGE_SIZE)),
+ ("/api/v1/nudges/33/responses.json", "0"),
+ ]