:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(warehouse_sources): add impact.com survey responses table (#119244)
Tom Owers73c83053ee0f
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txtindex 3d44d567ed91..0282f37f4431 100644--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt@@ -660,7 +660,6 @@1 products/warehouse_sources/backend/temporal/data_imports/sources/hyperspell/hyperspell.py1 products/warehouse_sources/backend/temporal/data_imports/sources/hyperspell/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/imagga/settings.py-1 products/warehouse_sources/backend/temporal/data_imports/sources/impact/impact.py1 products/warehouse_sources/backend/temporal/data_imports/sources/incident_io/incident_io.py1 products/warehouse_sources/backend/temporal/data_imports/sources/incident_io/settings.py1 products/warehouse_sources/backend/temporal/data_imports/sources/infisical/infisical.py@@ -1376,7 +1375,7 @@2 products/warehouse_sources/backend/temporal/data_imports/sources/common/typings.py2 products/warehouse_sources/backend/temporal/data_imports/sources/grafana/grafana.py2 products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/settings.py-2 products/warehouse_sources/backend/temporal/data_imports/sources/impact/settings.py+1 products/warehouse_sources/backend/temporal/data_imports/sources/impact/settings.py2 products/warehouse_sources/backend/temporal/data_imports/sources/jamf_pro/jamf_pro.py2 products/warehouse_sources/backend/temporal/data_imports/sources/linkedin_ads/linkedin_ads.py2 products/warehouse_sources/backend/temporal/data_imports/sources/mailchimp/settings.pydiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.mdindex 7f173dc456f1..9f19a3047446 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/COVERAGE_GAPS_APPENDIX.md@@ -6180,6 +6180,7 @@ Diffed against: <https://integrations.impact.com/llms-full.txt/4>- [ ] `Deals (GET /Advertisers/{AccountSID}/Deals)` — compare promotional offers and their effective dates with conversion performance (medium)- [ ] `Catalog Items (GET /Advertisers/{AccountSID}/Catalogs/{CatalogId}/Items)` — analyze affiliate product inventory, pricing, and availability by catalog (medium)- [ ] `Campaign Notes (GET /Advertisers/{AccountSID}/Campaigns/{CampaignId}/Notes)` — retain partner management activity and discussion context for campaign analysis (low)+- [x] `Survey responses (GET /Advertisers/{AccountSID}/Programs/{ProgramId}/Surveys/{SurveyId}/Responses)` — partner answers to program surveys, joinable to partners and campaigns (medium)Note: Parsed embedded OpenAPI blocks in the official paginated documentation, including Brand API v14 pages and nested collections. Contracts, action updates, and both invoice item tables are already covered. Click and report exports require job polling and downloads; action items and catalog items need parent fan-out. Submissions and configuration are excluded.diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/impact/canonical_descriptions.py b/products/warehouse_sources/backend/temporal/data_imports/sources/impact/canonical_descriptions.pyindex 5b37d8e511f6..c8504d7cf7f3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/impact/canonical_descriptions.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/impact/canonical_descriptions.py@@ -94,6 +94,24 @@ CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {"CampaignTerms": "Campaign-level payout term overrides.",},},+ "SurveyResponses": {+ "description": "Partner answers to program surveys. Impact keeps only the most recent answer per question per partner user.",+ "docs_url": "https://integrations.impact.com/brand-api-reference/reference/surveys-and-custom-fields/surveys",+ "columns": {+ "Id": "Identifier of the survey response.",+ "CampaignId": "Campaign (program) the survey belongs to.",+ "SurveyId": "Survey the response belongs to.",+ "QuestionId": "Survey question that was answered.",+ "Question": "Text of the survey question.",+ "PartnerId": "Partner who responded.",+ "PartnerName": "Name of the partner who responded.",+ "UserId": "Partner user who submitted the response.",+ "UserEmail": "Email address of the partner user who submitted the response.",+ "ResponseValue": "The answer. For list-based custom fields, a comma-separated list of enumeration IDs.",+ "ResponseDate": "When the response was submitted.",+ "CustomFieldId": "Custom field that defines what data the question collects.",+ },+ },"InvoiceLineItems": {"description": "Summary invoice line items, one row per campaign/description on an invoice, split out of the Invoices payload.","docs_url": "https://integrations.impact.com/brand-api-reference/reference/invoices/invoices",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/impact/impact.py b/products/warehouse_sources/backend/temporal/data_imports/sources/impact/impact.pyindex 391df0a7e292..66f81269e7a8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/impact/impact.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/impact/impact.py@@ -23,8 +23,10 @@ BASE_URL = "https://api.impact.com"REQUEST_TIMEOUT_SECONDS = 60+SURVEYS_PATH = "/Programs/{campaign_id}/Surveys"+[email protected](frozen=True)class ImpactResumeConfig:# Simple (non-fan-out) endpoints: the last page fetched. Re-fetched on resume (merge dedupes).page: Optional[int] = None@@ -32,6 +34,8 @@ class ImpactResumeConfig:campaign_id: Optional[int] = None# Actions: ISO start of the date window currently being processed.window_start: Optional[str] = None+ # SurveyResponses: the last survey whose responses were yielded, within `campaign_id`.+ survey_id: Optional[int] = Nonedef _get_session(account_sid: str, auth_token: str, api_version: str = IMPACT_API_VERSION_LEGACY) -> requests.Session:@@ -57,6 +61,18 @@ def _fetch(return response.json()+def _fetch_or_none(+ session: requests.Session, account_sid: str, path: str, params: dict[str, Any], logger: FilteringBoundLogger+) -> Any:+ try:+ return _fetch(session, account_sid, path, params, logger)+ except requests.HTTPError as e:+ if e.response is not None and e.response.status_code == 404:+ logger.warning(f"Impact: {path} returned 404, skipping")+ return None+ raise++def validate_credentials(account_sid: str, auth_token: str, api_version: str = IMPACT_API_VERSION_LEGACY) -> bool:try:session = _get_session(account_sid, auth_token, api_version)@@ -271,6 +287,58 @@ def _get_rows_fanout(resumable_source_manager.save_state(ImpactResumeConfig(campaign_id=campaign_id, window_start=window_start))+def _get_rows_survey_responses(+ session: requests.Session,+ account_sid: str,+ config: ImpactEndpointConfig,+ logger: FilteringBoundLogger,+ resumable_source_manager: ResumableSourceManager[ImpactResumeConfig],+) -> Iterator[list[dict[str, Any]]]:+ """Fetch the responses of every survey on every program (campaign).++ Neither the survey list nor the responses endpoint is paginated. A program or survey that+ returns 404 (removed mid-sync, or surveys unavailable on the account's API version) is skipped."""+ campaign_ids = _discover_campaign_ids(session, account_sid, logger)+ resume = resumable_source_manager.load_state() if resumable_source_manager.can_resume() else None+ resume_campaign_id = resume.campaign_id if resume else None+ resume_survey_id = resume.survey_id if resume else None++ for campaign_id in campaign_ids:+ if resume_campaign_id is not None and campaign_id < resume_campaign_id:+ continue+ surveys = _fetch_or_none(session, account_sid, SURVEYS_PATH.format(campaign_id=campaign_id), {}, logger)+ survey_rows = surveys.get("Surveys", []) if isinstance(surveys, dict) else []+ survey_ids = sorted(+ {+ survey_id+ for row in survey_rows+ if isinstance(row, dict) and (survey_id := _safe_int(row.get("Id"))) is not None+ }+ )+ if campaign_id == resume_campaign_id and resume_survey_id is not None:+ survey_ids = [survey_id for survey_id in survey_ids if survey_id > resume_survey_id]+ # Campaign ids are walked in ascending order, so the next id marks this whole program as done.+ program_done = ImpactResumeConfig(campaign_id=campaign_id + 1)+ if not survey_ids:+ resumable_source_manager.save_state(program_done)+ resumable_source_manager.safe_point()+ continue+ for index, survey_id in enumerate(survey_ids):+ path = config.path.format(campaign_id=campaign_id, survey_id=survey_id)+ rows = _rows_from_response(config, _fetch_or_none(session, account_sid, path, {}, logger))+ is_last_survey = index == len(survey_ids) - 1+ resumable_source_manager.save_state(+ program_done if is_last_survey else ImpactResumeConfig(campaign_id=campaign_id, survey_id=survey_id)+ )+ if rows:+ for row in rows:+ row["CampaignId"] = campaign_id+ row["SurveyId"] = survey_id+ yield rows+ else:+ resumable_source_manager.safe_point()++def _get_rows_nested(session: requests.Session,account_sid: str,@@ -326,6 +394,10 @@ def get_rows(yield from _get_rows_nested(session, account_sid, config, logger, resumable_source_manager)return+ if config.requires_survey_fanout:+ yield from _get_rows_survey_responses(session, account_sid, config, logger, resumable_source_manager)+ return+if config.requires_campaign_fanout:yield from _get_rows_fanout(session,diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/impact/settings.py b/products/warehouse_sources/backend/temporal/data_imports/sources/impact/settings.pyindex 31b7c34bef78..9f277a040d1e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/impact/settings.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/impact/settings.py@@ -48,7 +48,7 @@ class NestedTableConfig:line_number_field: str = "LineNumber"-@dataclass+@dataclass(frozen=True)class ImpactEndpointConfig:name: str# Path relative to https://api.impact.com/Advertisers/{account_sid}. When campaign_id_in_path is@@ -64,6 +64,9 @@ class ImpactEndpointConfig:# Contracts nests the campaign in the path (/Campaigns/{campaign_id}/Contracts) and, unlike the# action endpoints, doesn't echo it back on each row, so the fan-out injects CampaignId itself.campaign_id_in_path: bool = False+ # Survey responses sit two levels down (program -> survey -> responses), so the fan-out lists each+ # program's surveys first. `path` then takes both `{campaign_id}` and `{survey_id}`.+ requires_survey_fanout: bool = False# The action endpoints cap their date filter at a 45-day span, so history is walked in bounded# windows.date_windowed: bool = False@@ -173,6 +176,18 @@ IMPACT_ENDPOINTS: dict[str, ImpactEndpointConfig] = {"Impact returns active contracts by default. Full refresh only."),),+ "SurveyResponses": ImpactEndpointConfig(+ name="SurveyResponses",+ path="/Programs/{campaign_id}/Surveys/{survey_id}/Responses",+ data_key="Responses",+ primary_keys=["SurveyId", "Id"],+ requires_survey_fanout=True,+ description=(+ "Partner answers to your program surveys, one row per question per partner user (Impact "+ "keeps only the latest answer). Fetched per survey on every campaign. Requires API "+ "version 14. Full refresh only."+ ),+ ),"InvoiceLineItems": ImpactEndpointConfig(name="InvoiceLineItems",path="/Invoices",diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/impact/tests/test_impact.py b/products/warehouse_sources/backend/temporal/data_imports/sources/impact/tests/test_impact.pyindex 22852644b2eb..b4b592e4c39b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/impact/tests/test_impact.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/impact/tests/test_impact.py@@ -207,6 +207,54 @@ class TestGetRowsContractsFanout:assert injected == {10, 20}+class TestGetRowsSurveyResponses:+ @parameterized.expand(+ [+ (+ "full_walk",+ None,+ [+ "/Advertisers/sid/Programs/10/Surveys/1/Responses",+ "/Advertisers/sid/Programs/10/Surveys/2/Responses",+ ],+ [{"Id": 5, "CampaignId": 10, "SurveyId": 1}, {"Id": 6, "CampaignId": 10, "SurveyId": 2}],+ ),+ (+ "resume_after_bookmarked_survey",+ ImpactResumeConfig(campaign_id=10, survey_id=1),+ ["/Advertisers/sid/Programs/10/Surveys/2/Responses"],+ [{"Id": 6, "CampaignId": 10, "SurveyId": 2}],+ ),+ ("resume_after_completed_program", ImpactResumeConfig(campaign_id=20), [], []),+ ]+ )+ def test_walks_programs_and_surveys(+ self,+ _name: str,+ resume_state: Optional[ImpactResumeConfig],+ expected_response_paths: list[str],+ expected_rows: list[dict[str, Any]],+ ) -> None:+ result = _driver().run(+ "SurveyResponses",+ route(+ {+ "/Campaigns": [ScriptedResponse(json={"Campaigns": [{"Id": 20}, {"Id": 10}], "@numpages": "1"})],+ "/Programs/10/Surveys": [ScriptedResponse(json={"Surveys": [{"Id": 2}, {"Id": 1}]})],+ "/Programs/20/Surveys": [ScriptedResponse(status=404)],+ "/Surveys/1/Responses": [ScriptedResponse(json={"Responses": [{"Id": 5}]})],+ "/Surveys/2/Responses": [ScriptedResponse(json={"Responses": [{"Id": 6}]})],+ }+ ),+ resume_state=resume_state,+ )+ assert result.raised is None+ assert [path for path in result.paths if path.endswith("/Responses")] == expected_response_paths+ assert "/Advertisers/sid/Programs/20/Surveys" in result.paths+ assert result.rows == expected_rows+ assert result.saved_states[-1] == ImpactResumeConfig(campaign_id=21)++class TestImpactSourceResponse:@parameterized.expand([