:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): add 1 missing table to the Qualaroo source (#120840)
Tom Owers3e0cfb97dc47
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex a80e3be9c6ec..9370c56295eb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -9924,7 +9924,7 @@ Today (1): `nudges`Diffed against: <https://help.qualaroo.com/hc/en-us/articles/201969438-The-REST-Reporting-API>-- [ ] `GET /nudges/{nudge_id}/responses.json` — the survey answers themselves - the only analytical data the Reporting API exposes; today the source syncs survey definitions with no responses (high)+- [x] `GET /nudges/{nudge_id}/responses.json` — the survey answers themselves - the only analytical data the Reporting API exposes; today the source syncs survey definitions with no responses (high)Note: The REST Reporting API is essentially two endpoints (nudges list, and per-nudge responses with start_date/end_date/offset/limit paging). The source's settings define only the nudges endpoint - no dynamic table discovery - so the entire response dataset is unavailable. Adding it requires a fan-out over nudge IDs, same shape as other per-parent sources.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/canonical_descriptions.pyindex 80c10c0fc7c2..87cc2ed23184 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/canonical_descriptions.py@@ -19,4 +19,21 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"updated_at": "When the nudge was last updated.",},},+ "responses": {+ "description": "One respondent's response to a Qualaroo nudge (survey), with their answers and metadata.",+ "docs_url": "https://help.qualaroo.com/hc/en-us/articles/201969438-The-REST-Reporting-API",+ "columns": {+ "id": "The ID of the response record.",+ "nudge_id": "The ID of the nudge (survey) that the response belongs to.",+ "time": "When the response was recorded, in the account time zone with its UTC offset.",+ "token": "An anonymous identifier of the respondent.",+ "ip_address": "The IP address of the respondent.",+ "identity": "The identity that you tagged the respondent with.",+ "page": "The page of your website where the respondent interacted with the survey.",+ "referrer": "The URL of the referrer page, if there was one.",+ "user_agent": "The User-Agent of the respondent's browser.",+ "answered_questions": "A map of the questions that the respondent answered, with each question ID, title, answer text, selected option ID and comment.",+ "properties": "The custom name/value properties that you tagged the respondent with.",+ },+ },}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/qualaroo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/qualaroo.pyindex 6c2401c72efe..32387338a40c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/qualaroo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/qualaroo.py@@ -1,34 +1,137 @@import dataclasses+from datetime import UTC, datetime, timedeltafrom typing import Any, Optionalfrom requests.auth import HTTPBasicAuth+from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_valuefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_sessionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (RESTAPIConfig,rest_api_resource,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ build_dependent_resource,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (OffsetPaginator,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (+ ClientConfig,+ IncrementalConfig,+)from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManagerfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse-from products.warehouse_sources.backend.temporal.data_imports.sources.qualaroo.settings import QUALAROO_ENDPOINTS+from products.warehouse_sources.backend.temporal.data_imports.sources.qualaroo.settings import (+ PAGE_SIZE,+ QUALAROO_ENDPOINTS,+)QUALAROO_BASE_URL = "https://api.qualaroo.com/api/v1"-# The Reporting API caps a page at 500 records; the largest page minimises round trips.-PAGE_SIZE = 500# Cheap endpoint used to confirm the credentials are genuine. The API key/secret pair is# account-wide, so one probe validates access to every list endpoint.DEFAULT_PROBE_PATH = "/nudges.json"[email protected](frozen=False)class QualarooResumeConfig:# Offset of the next page to fetch. Offset pagination is deterministic, so a crashed# full-refresh sync resumes from the page after the last one yielded; merge dedupes on `id`.offset: int = 0+ # Progress of an endpoint that is listed once per nudge: the nudges that are done, the nudge in+ # progress, and the page cursor of that nudge. The shared fan-out helper owns the shape.+ fanout_state: Optional[dict[str, Any]] = None+ # `end_date` fixed at sync start, so responses that arrive mid fan-out are left for the next run.+ sync_end: Optional[str] = None+++def _client_config(api_key: str, api_secret: str) -> ClientConfig:+ return {+ "base_url": QUALAROO_BASE_URL,+ "headers": {"Accept": "application/json"},+ # Qualaroo uses HTTP Basic auth with the API key as username and the secret as password.+ "auth": {"type": "http_basic", "username": api_key, "password": api_secret},+ # Qualaroo has no top-level `total`; termination is a short/empty page (OffsetPaginator default).+ "paginator": OffsetPaginator(limit=PAGE_SIZE, total_path=None),+ }+++def _to_unix_seconds(value: Any) -> Optional[int]:+ parsed = parse_datetime_value(value)+ if parsed is None:+ # No watermark yet: omit `start_date`, so the API starts from the first response.+ return None+ # The docs do not say whether `start_date` is inclusive. A one-second overlap keeps rows that+ # share the watermark second, and merge dedupes the rows it reads again.+ return int((parsed - timedelta(seconds=1)).timestamp())+++def _incremental_window(cursor_path: str, sync_end: str) -> IncrementalConfig:+ return {+ "cursor_path": cursor_path,+ "start_param": "start_date",+ "end_param": "end_date",+ "end_value": sync_end,+ "convert": _to_unix_seconds,+ }+++def _fanout_source(+ api_key: str,+ api_secret: str,+ endpoint: str,+ team_id: int,+ job_id: str,+ resumable_source_manager: ResumableSourceManager[QualarooResumeConfig],+ should_use_incremental_field: bool,+ db_incremental_field_last_value: Optional[Any],+ incremental_field: Optional[str],+) -> SourceResponse:+ config = QUALAROO_ENDPOINTS[endpoint]+ assert config.fanout is not None++ initial_fanout_state: Optional[dict[str, Any]] = None+ sync_end: Optional[str] = None+ if resumable_source_manager.can_resume():+ resume = resumable_source_manager.load_state()+ if resume is not None:+ initial_fanout_state = resume.fanout_state+ sync_end = resume.sync_end+ run_sync_end = sync_end or datetime.now(UTC).isoformat()++ def save_fanout_checkpoint(state: Optional[dict[str, Any]]) -> None:+ if state is not None:+ resumable_source_manager.save_state(QualarooResumeConfig(fanout_state=state, sync_end=run_sync_end))++ resource = build_dependent_resource(+ endpoint_configs=QUALAROO_ENDPOINTS,+ child_endpoint=endpoint,+ fanout=config.fanout,+ # Survey answers are free text that the sample scrubbers cannot reliably redact.+ client_config={**_client_config(api_key, api_secret), "capture": False},+ path_format_values={},+ team_id=team_id,+ job_id=job_id,+ db_incremental_field_last_value=db_incremental_field_last_value,+ should_use_incremental_field=should_use_incremental_field,+ incremental_field=incremental_field,+ incremental_config_factory=lambda cursor_path: _incremental_window(cursor_path, run_sync_end),+ # The client OffsetPaginator already sends `limit` on every request.+ page_size_param=None,+ parent_endpoint_extra={"data_selector_malformed_retryable": True},+ child_endpoint_extra={"data_selector_malformed_retryable": True},+ resume_hook=save_fanout_checkpoint,+ initial_paginator_state=initial_fanout_state,+ )++ return SourceResponse(+ name=endpoint,+ items=lambda: resource,+ primary_keys=config.primary_keys,+ partition_count=1,+ partition_size=1,+ )def qualaroo_source(@@ -38,19 +141,26 @@ def qualaroo_source(team_id: int,job_id: str,resumable_source_manager: ResumableSourceManager[QualarooResumeConfig],+ should_use_incremental_field: bool = False,db_incremental_field_last_value: Optional[Any] = None,+ incremental_field: Optional[str] = None,) -> SourceResponse:config = QUALAROO_ENDPOINTS[endpoint]+ if config.fanout is not None:+ return _fanout_source(+ api_key,+ api_secret,+ endpoint,+ team_id,+ job_id,+ resumable_source_manager,+ should_use_incremental_field,+ db_incremental_field_last_value,+ incremental_field,+ )rest_config: RESTAPIConfig = {- "client": {- "base_url": QUALAROO_BASE_URL,- "headers": {"Accept": "application/json"},- # Qualaroo uses HTTP Basic auth with the API key as username and the secret as password.- "auth": {"type": "http_basic", "username": api_key, "password": api_secret},- # Qualaroo has no top-level `total`; termination is a short/empty page (OffsetPaginator default).- "paginator": OffsetPaginator(limit=PAGE_SIZE, total_path=None),- },+ "client": _client_config(api_key, api_secret),# Per-resource settings are fully specified below, so no shared defaults are needed."resource_defaults": {},"resources": [diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/settings.pyindex ff75f8cdbda5..e55164398776 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/settings.py@@ -1,27 +1,66 @@from dataclasses import dataclass, field+from typing import Optional-from products.warehouse_sources.backend.types import IncrementalField+from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (+ DependentEndpointConfig,+)+from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType+# The Reporting API caps a page at 500 records; the largest page minimises round trips.+PAGE_SIZE = 500-@dataclass++@dataclass(frozen=False)class QualarooEndpointConfig:name: strpath: str# Qualaroo nudge (survey) IDs are unique within an account, so `id` is a safe primary key.primary_keys: list[str] = field(default_factory=lambda: ["id"])+ page_size: int = PAGE_SIZE+ incremental_fields: list[IncrementalField] = field(default_factory=list)+ default_incremental_field: Optional[str] = None+ fanout: Optional[DependentEndpointConfig] = None-# Qualaroo REST Reporting API top-level list endpoints. Only account-wide lists are exposed here:-# per-survey responses live at /nudges/{id}/responses.json and require a parent nudge ID, so they-# are a fan-out stream and are intentionally skipped for now.+# Qualaroo REST Reporting API endpoints. `nudges` is the account-wide survey list. `responses` is+# listed once per nudge at /nudges/{nudge_id}/responses.json.#-# All endpoints are full-refresh only. The responses endpoint supports server-side start_date/-# end_date filters, but the nudges list has no reliably ordered timestamp filter, so there is no-# incremental cursor to advance (see the implementing-warehouse-sources skill).+# The nudges list has no server-side timestamp filter, so it is full refresh only. Responses take+# a `start_date` filter in UNIX seconds against the submission `time`, so they sync incrementally.QUALAROO_ENDPOINTS: dict[str, QualarooEndpointConfig] = {"nudges": QualarooEndpointConfig(name="nudges", path="/nudges.json"),+ # The docs do not say that a response id is unique across nudges, so the nudge id is part of the key.+ "responses": QualarooEndpointConfig(+ name="responses",+ path="/nudges/{nudge_id}/responses.json",+ primary_keys=["nudge_id", "id"],+ incremental_fields=[+ {+ "label": "time",+ "type": IncrementalFieldType.DateTime,+ "field": "time",+ "field_type": IncrementalFieldType.DateTime,+ }+ ],+ default_incremental_field="time",+ fanout=DependentEndpointConfig(+ parent_name="nudges",+ resolve_param="nudge_id",+ resolve_field="id",+ include_from_parent=["id"],+ parent_field_renames={"id": "nudge_id"},+ # Without `order=asc` the API returns newest first, so a response submitted mid-sync+ # shifts the offset pages and one row is read twice while another is skipped.+ child_params={"order": "asc"},+ # The API answers 404 for a nudge deleted between the nudge listing and this request.+ # That nudge has no responses to sync, so it must not fail the nudges that remain.+ child_response_actions=[{"status_code": 404, "action": "ignore"}],+ ),+ ),}ENDPOINTS = tuple(QUALAROO_ENDPOINTS.keys())-INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {}+INCREMENTAL_FIELDS: dict[str, list[IncrementalField]] = {+ name: config.incremental_fields for name, config in QUALAROO_ENDPOINTS.items() if config.incremental_fields+}diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/source.pyindex 6f553e927299..a7bafabf0358 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/source.py@@ -107,10 +107,9 @@ You can find your API key and secret under **Settings → API** in [Qualaroo](htforce_refresh: bool = False,api_version: str | None = None,) -> list[SourceSchema]:- # Every endpoint is full refresh only — the nudges list exposes no reliably ordered- # server-side timestamp filter, so INCREMENTAL_FIELDS is empty and every schema is- # non-incremental / non-append.- return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)+ # Responses are merge only: `start_date` re-reads the watermark second, which append would+ # store twice.+ return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=("responses",))def validate_credentials(self,@@ -141,5 +140,9 @@ You can find your API key and secret under **Settings → API** in [Qualaroo](htteam_id=inputs.team_id,job_id=inputs.job_id,resumable_source_manager=resumable_source_manager,- db_incremental_field_last_value=None, # every Qualaroo endpoint is full refresh+ should_use_incremental_field=inputs.should_use_incremental_field,+ db_incremental_field_last_value=(+ inputs.db_incremental_field_last_value if inputs.should_use_incremental_field else None+ ),+ incremental_field=inputs.incremental_field,)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/tests/test_qualaroo.py b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/tests/test_qualaroo.pyindex 5707abd2eb69..722377e3124e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/tests/test_qualaroo.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/qualaroo/tests/test_qualaroo.py@@ -1,3 +1,4 @@+from datetime import UTC, datetimefrom typing import Anyimport requests@@ -12,6 +13,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tesScriptedResponse,SourceDriver,always,+ route,scripted_network,)from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.qualaroo import (@@ -113,10 +115,129 @@ class TestSourceResponse:assert result.response is not Noneassert result.response.name == endpoint- assert result.response.primary_keys == ["id"]+ assert result.response.primary_keys == QUALAROO_ENDPOINTS[endpoint].primary_keys# No stable creation timestamp is guaranteed across every object, so we don't partition.assert result.response.partition_mode is None- def test_every_endpoint_uses_id_primary_key(self) -> None:- assert all(config.primary_keys == ["id"] for config in QUALAROO_ENDPOINTS.values())- assert set(QUALAROO_ENDPOINTS) == set(ENDPOINTS)++class TestResponsesFanOut:+ def test_walks_every_nudge_and_every_response_page(self) -> None:+ result = _driver().run(+ "responses",+ route(+ {+ "/nudges.json": [ScriptedResponse(json=[{"id": 11}, {"id": 22}, {"id": 33}])],+ "/nudges/11/responses.json": [+ ScriptedResponse(+ json=[{"id": i, "time": "2024-03-01 10:00:00 -0700"} for i in range(PAGE_SIZE)]+ ),+ ScriptedResponse(json=[{"id": 900, "answered_questions": {}, "properties": None}]),+ ],+ "/nudges/22/responses.json": [ScriptedResponse(status=404, json={"error": "Not Found"})],+ "/nudges/33/responses.json": [ScriptedResponse(json=[{"id": 1}])],+ }+ ),+ )++ assert result.raised is None+ assert len(result.rows) == PAGE_SIZE + 2+ assert result.rows[PAGE_SIZE] == {"id": 900, "answered_questions": {}, "properties": None, "nudge_id": 11}+ # Response ids are keyed with their nudge, so the same id under two nudges stays two rows.+ assert {(row["nudge_id"], row["id"]) for row in result.rows if row["id"] == 1} == {(11, 1), (33, 1)}+ assert [(request.path, request.param("offset")) for request in result.requests] == [+ ("/api/v1/nudges.json", "0"),+ ("/api/v1/nudges/11/responses.json", "0"),+ ("/api/v1/nudges/11/responses.json", str(PAGE_SIZE)),+ ("/api/v1/nudges/22/responses.json", "0"),+ ("/api/v1/nudges/33/responses.json", "0"),+ ]+ child_requests = [request for request in result.requests if request.path.endswith("/responses.json")]+ assert {request.param("order") for request in child_requests} == {"asc"}+ assert {request.param("limit") for request in result.requests} == {str(PAGE_SIZE)}++ @parameterized.expand(+ [+ ("full_refresh", {"should_use_incremental_field": False}, None),+ ("first_incremental_run", {"incremental_field": "time"}, None),+ (+ "watermark_string_with_offset",+ {"incremental_field": "time", "db_incremental_field_last_value": "2024-03-01 10:00:00 -0700"},+ str(int(datetime(2024, 3, 1, 17, 0, 0, tzinfo=UTC).timestamp()) - 1),+ ),+ (+ "watermark_naive_datetime_is_utc",+ {"incremental_field": "time", "db_incremental_field_last_value": datetime(2024, 3, 1, 17, 0, 0)},+ str(int(datetime(2024, 3, 1, 17, 0, 0, tzinfo=UTC).timestamp()) - 1),+ ),+ ]+ )+ def test_start_date_follows_the_watermark(+ self, _name: str, inputs: dict[str, Any], expected_start_date: str | None+ ) -> None:+ result = _driver().run(+ "responses",+ route(+ {+ "/nudges.json": [ScriptedResponse(json=[{"id": 11}])],+ "/nudges/11/responses.json": [ScriptedResponse(json=[{"id": 1}])],+ }+ ),+ **inputs,+ )++ assert result.raised is None+ assert [request.param("start_date") for request in result.requests] == [None, expected_start_date]+ end_dates = [request.param("end_date") for request in result.requests]+ assert end_dates[0] is None+ assert (end_dates[1] is not None) == inputs.get("should_use_incremental_field", True)++ def test_resumed_incremental_sync_keeps_the_original_end_date(self) -> None:+ sync_end = "2024-03-02T00:00:00+00:00"+ result = _driver().run(+ "responses",+ route(+ {+ "/nudges.json": [ScriptedResponse(json=[{"id": 11}, {"id": 22}])],+ "/nudges/22/responses.json": [ScriptedResponse(json=[{"id": 7, "time": "2024-03-01 10:00:00"}])],+ }+ ),+ resume_state=QualarooResumeConfig(+ fanout_state={"completed": ["/nudges/11/responses.json"], "current": None, "child_state": None},+ sync_end=sync_end,+ ),+ incremental_field="time",+ db_incremental_field_last_value="2024-03-01 00:00:00",+ )++ assert result.raised is None+ child_requests = [request for request in result.requests if request.path.endswith("/responses.json")]+ expected_end_date = str(int(datetime(2024, 3, 2, tzinfo=UTC).timestamp()) - 1)+ assert [request.param("end_date") for request in child_requests] == [expected_end_date]+ assert all(state.sync_end == sync_end for state in result.saved_states)++ def test_resumes_inside_the_nudge_in_progress(self) -> None:+ result = _driver().run(+ "responses",+ route(+ {+ "/nudges.json": [ScriptedResponse(json=[{"id": 11}, {"id": 22}, {"id": 33}])],+ "/nudges/22/responses.json": [ScriptedResponse(json=[{"id": 7}])],+ "/nudges/33/responses.json": [ScriptedResponse(json=[])],+ }+ ),+ resume_state=QualarooResumeConfig(+ fanout_state={+ "completed": ["/nudges/11/responses.json"],+ "current": "/nudges/22/responses.json",+ "child_state": {"offset": PAGE_SIZE},+ }+ ),+ )++ assert result.raised is None+ assert result.rows == [{"id": 7, "nudge_id": 22}]+ assert [(request.path, request.param("offset")) for request in result.requests] == [+ ("/api/v1/nudges.json", "0"),+ ("/api/v1/nudges/22/responses.json", str(PAGE_SIZE)),+ ("/api/v1/nudges/33/responses.json", "0"),+ ]