Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(data-warehouse): move webhook vendors to the plugin folder (#128249)

Tom Owers5313c2b16903
diff --git a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
index 37faf0557fea..b554d1bebe0a 100644
--- a/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
+++ b/posthog/test/repo_invariants/dataclass_frozen_baseline.txt
@@ -219,18 +219,18 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/utils.py
1 products/warehouse_sources/sources/browser_use/browser_use.py
1 products/warehouse_sources/sources/browser_use/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/calendly/calendly.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/calendly/settings.py
+1 products/warehouse_sources/sources/calendly/calendly.py
+1 products/warehouse_sources/sources/calendly/settings.py
1 products/warehouse_sources/sources/campaign_monitor/campaign_monitor.py
1 products/warehouse_sources/sources/campayn/settings.py
1 products/warehouse_sources/sources/capsule_crm/capsule_crm.py
1 products/warehouse_sources/sources/care_quality_commission/care_quality_commission.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/chatwoot.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/settings.py
+1 products/warehouse_sources/sources/chatwoot/chatwoot.py
+1 products/warehouse_sources/sources/chatwoot/settings.py
1 products/warehouse_sources/sources/checkout_com/checkout_com.py
1 products/warehouse_sources/sources/circleci_insights/circleci_insights.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py
+1 products/warehouse_sources/sources/clerk/clerk.py
+1 products/warehouse_sources/sources/clerk/settings.py
1 products/warehouse_sources/sources/clickhouse_cloud/clickhouse_cloud.py
1 products/warehouse_sources/sources/coda/settings.py
1 products/warehouse_sources/sources/coin_api/coin_api.py
@@ -246,15 +246,15 @@
1 products/warehouse_sources/sources/docuseal/settings.py
1 products/warehouse_sources/sources/dropbox_sign/dropbox_sign.py
1 products/warehouse_sources/sources/exchange_rates_api/exchange_rates_api.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/featurebase.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/settings.py
+1 products/warehouse_sources/sources/featurebase/featurebase.py
+1 products/warehouse_sources/sources/featurebase/settings.py
1 products/warehouse_sources/sources/financial_modelling/financial_modelling.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/fourthwall.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/settings.py
+1 products/warehouse_sources/sources/fourthwall/fourthwall.py
+1 products/warehouse_sources/sources/fourthwall/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/freshdesk/freshdesk.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/fusionauth/fusionauth.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/gitea/gitea.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/gitea/settings.py
+1 products/warehouse_sources/sources/gitea/gitea.py
+1 products/warehouse_sources/sources/gitea/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/github/github.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/github/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/google_ads/configs.py
@@ -262,14 +262,14 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/google_search_console/google_search_console.py
1 products/warehouse_sources/sources/google_webfonts/settings.py
1 products/warehouse_sources/sources/gridly/gridly.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/gumroad.py
+1 products/warehouse_sources/sources/gumroad/gumroad.py
1 products/warehouse_sources/sources/height/settings.py
1 products/warehouse_sources/sources/hellobaton/settings.py
1 products/warehouse_sources/sources/hoorayhr/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/hubspot.py
1 products/warehouse_sources/sources/inflowinventory/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/instantly/instantly.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/instantly/settings.py
+1 products/warehouse_sources/sources/instantly/instantly.py
+1 products/warehouse_sources/sources/instantly/settings.py
1 products/warehouse_sources/sources/ip2whois/settings.py
1 products/warehouse_sources/sources/jfrog_artifactory/jfrog_artifactory.py
1 products/warehouse_sources/sources/judgeme_reviews/judgeme_reviews.py
@@ -280,20 +280,20 @@
1 products/warehouse_sources/sources/kong_konnect/kong_konnect.py
1 products/warehouse_sources/sources/lambda_labs/lambda_labs.py
1 products/warehouse_sources/sources/leadfeeder/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/lemon_squeezy.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/settings.py
+1 products/warehouse_sources/sources/lemon_squeezy/lemon_squeezy.py
+1 products/warehouse_sources/sources/lemon_squeezy/settings.py
1 products/warehouse_sources/sources/lingo_dev/lingo_dev.py
1 products/warehouse_sources/sources/lingo_dev/settings.py
1 products/warehouse_sources/sources/logz_io/logz_io.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/mailerlite.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/mailjet.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/settings.py
+1 products/warehouse_sources/sources/mailerlite/mailerlite.py
+1 products/warehouse_sources/sources/mailerlite/settings.py
+1 products/warehouse_sources/sources/mailgun/mailgun.py
+1 products/warehouse_sources/sources/mailjet/mailjet.py
+1 products/warehouse_sources/sources/mailjet/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/matomo/matomo.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/matomo/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/mem0/mem0.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/mem0/settings.py
+1 products/warehouse_sources/sources/mem0/mem0.py
+1 products/warehouse_sources/sources/mem0/settings.py
1 products/warehouse_sources/sources/mention/settings.py
1 products/warehouse_sources/sources/monte_carlo/monte_carlo.py
1 products/warehouse_sources/sources/new_relic/new_relic.py
@@ -311,16 +311,16 @@
1 products/warehouse_sources/sources/papersign/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pinterest_ads/pinterest_ads.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/pinterest_ads/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/pipedrive.py
+1 products/warehouse_sources/sources/pipedrive/pipedrive.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/plunk/plunk.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/plunk/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/postgres/postgres.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/postmark/postmark.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/postmark/settings.py
+1 products/warehouse_sources/sources/postmark/postmark.py
+1 products/warehouse_sources/sources/postmark/settings.py
1 products/warehouse_sources/sources/qualys_vmdr/qualys_vmdr.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/reddit_ads/settings.py
1 products/warehouse_sources/sources/rentcast/rentcast.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py
+1 products/warehouse_sources/sources/revenuecat/revenuecat.py
1 products/warehouse_sources/sources/rocketlane/rocketlane.py
1 products/warehouse_sources/sources/rocketlane/settings.py
1 products/warehouse_sources/sources/sage_hr/sage_hr.py
@@ -335,16 +335,16 @@
1 products/warehouse_sources/backend/temporal/data_imports/sources/sentry/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/servicenow/settings.py
1 products/warehouse_sources/sources/simfin/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/slack/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/slack/slack.py
+1 products/warehouse_sources/sources/slack/settings.py
+1 products/warehouse_sources/sources/slack/slack.py
1 products/warehouse_sources/sources/smaily/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/settings.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/snapchat_ads.py
1 products/warehouse_sources/sources/sonar_cloud/sonar_cloud.py
1 products/warehouse_sources/sources/sonatype_nexus/settings.py
1 products/warehouse_sources/sources/sonatype_nexus/sonatype_nexus.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/sparkpost.py
+1 products/warehouse_sources/sources/sparkpost/settings.py
+1 products/warehouse_sources/sources/sparkpost/sparkpost.py
1 products/warehouse_sources/sources/split_io/split_io.py
1 products/warehouse_sources/sources/splunk_observability_cloud/splunk_observability_cloud.py
1 products/warehouse_sources/sources/spot_io/spot_io.py
@@ -365,21 +365,21 @@
1 products/warehouse_sources/sources/twelve_labs/twelve_labs.py
1 products/warehouse_sources/sources/uk_companies_house/uk_companies_house.py
1 products/warehouse_sources/sources/unleash/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/uppromote.py
+1 products/warehouse_sources/sources/uppromote/uppromote.py
1 products/warehouse_sources/sources/upstash/settings.py
1 products/warehouse_sources/sources/veeqo/settings.py
1 products/warehouse_sources/sources/veeqo/veeqo.py
1 products/warehouse_sources/sources/vultr/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/webflow/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/webflow/webflow.py
+1 products/warehouse_sources/sources/webflow/settings.py
+1 products/warehouse_sources/sources/webflow/webflow.py
1 products/warehouse_sources/sources/weights_and_biases/weights_and_biases.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/whop/whop.py
+1 products/warehouse_sources/sources/whop/whop.py
1 products/warehouse_sources/sources/wikipedia_pageviews/wikipedia_pageviews.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/woocommerce.py
+1 products/warehouse_sources/sources/woocommerce/woocommerce.py
1 products/warehouse_sources/sources/world_bank/world_bank.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/yousign/settings.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/yousign/yousign.py
-1 products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py
+1 products/warehouse_sources/sources/yousign/settings.py
+1 products/warehouse_sources/sources/yousign/yousign.py
+1 products/warehouse_sources/sources/zapsign/zapsign.py
1 products/warehouse_sources/backend/temporal/data_imports/sources/zendesk/settings.py
1 products/warehouse_sources/sources/zendesk_sell/zendesk_sell.py
1 products/warehouse_sources/sources/zendesk_sunshine/zendesk_sunshine.py
diff --git a/posthog/test/repo_invariants/source_test_hand_written_status_error_baseline.txt b/posthog/test/repo_invariants/source_test_hand_written_status_error_baseline.txt
index f80aa4528c49..8c11d95c4133 100644
--- a/posthog/test/repo_invariants/source_test_hand_written_status_error_baseline.txt
+++ b/posthog/test/repo_invariants/source_test_hand_written_status_error_baseline.txt
@@ -1,7 +1,7 @@
-products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly_source.py
+products/warehouse_sources/sources/instantly/tests/test_instantly_source.py
products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_stripe_source.py
products/warehouse_sources/backend/temporal/data_imports/sources/tiktok_ads/test/test_tiktok_ads_source.py
-products/warehouse_sources/backend/temporal/data_imports/sources/whop/tests/test_whop_source.py
+products/warehouse_sources/sources/whop/tests/test_whop_source.py
products/warehouse_sources/sources/adobe_analytics/tests/test_adobe_analytics_source.py
products/warehouse_sources/sources/alguna/tests/test_alguna_source.py
products/warehouse_sources/sources/assemblyai/tests/test_assemblyai_source.py
diff --git a/posthog/test/repo_invariants/source_test_mocked_resume_manager_baseline.txt b/posthog/test/repo_invariants/source_test_mocked_resume_manager_baseline.txt
index 0fd8de29594e..84f4f4b7180a 100644
--- a/posthog/test/repo_invariants/source_test_mocked_resume_manager_baseline.txt
+++ b/posthog/test/repo_invariants/source_test_mocked_resume_manager_baseline.txt
@@ -1,15 +1,15 @@
products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads.py
products/warehouse_sources/backend/temporal/data_imports/sources/bing_ads/tests/test_bing_ads_source.py
-products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/tests/test_mailerlite.py
-products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/tests/test_mailerlite_source.py
+products/warehouse_sources/sources/mailerlite/tests/test_mailerlite.py
+products/warehouse_sources/sources/mailerlite/tests/test_mailerlite_source.py
products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
-products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.py
+products/warehouse_sources/sources/paymongo/tests/test_paymongo.py
products/warehouse_sources/backend/temporal/data_imports/sources/sentry/tests/test_sentry.py
-products/warehouse_sources/backend/temporal/data_imports/sources/slack/test_slack.py
+products/warehouse_sources/sources/slack/test_slack.py
products/warehouse_sources/backend/temporal/data_imports/sources/snapchat_ads/tests/test_snapchat_ads.py
-products/warehouse_sources/backend/temporal/data_imports/sources/webflow/tests/test_webflow.py
-products/warehouse_sources/backend/temporal/data_imports/sources/webflow/tests/test_webflow_source.py
-products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/tests/test_woocommerce_source.py
+products/warehouse_sources/sources/webflow/tests/test_webflow.py
+products/warehouse_sources/sources/webflow/tests/test_webflow_source.py
+products/warehouse_sources/sources/woocommerce/tests/test_woocommerce_source.py
products/warehouse_sources/backend/temporal/data_imports/sources/zendesk/tests/test_zendesk.py
products/warehouse_sources/sources/asknicely/tests/test_asknicely_source.py
products/warehouse_sources/sources/aws_batch/tests/test_aws_batch.py
diff --git a/posthog/test/repo_invariants/source_test_patched_session_baseline.txt b/posthog/test/repo_invariants/source_test_patched_session_baseline.txt
index 33ac1efb5d1e..ecaf61c2e6e3 100644
--- a/posthog/test/repo_invariants/source_test_patched_session_baseline.txt
+++ b/posthog/test/repo_invariants/source_test_patched_session_baseline.txt
@@ -1,9 +1,9 @@
-products/warehouse_sources/backend/temporal/data_imports/sources/calendly/tests/test_calendly.py
-products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/tests/test_chatwoot.py
+products/warehouse_sources/sources/calendly/tests/test_calendly.py
+products/warehouse_sources/sources/chatwoot/tests/test_chatwoot.py
products/warehouse_sources/backend/temporal/data_imports/sources/custom/test_custom_source.py
-products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/tests/test_featurebase.py
-products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/tests/test_fourthwall.py
-products/warehouse_sources/backend/temporal/data_imports/sources/gitea/tests/test_gitea.py
+products/warehouse_sources/sources/featurebase/tests/test_featurebase.py
+products/warehouse_sources/sources/fourthwall/tests/test_fourthwall.py
+products/warehouse_sources/sources/gitea/tests/test_gitea.py
products/warehouse_sources/backend/temporal/data_imports/sources/github/tests/test_github_api_version_headers.py
products/warehouse_sources/backend/temporal/data_imports/sources/github/tests/test_github_fetch_page.py
products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/test/test_hubspot_auth.py
@@ -11,13 +11,13 @@ products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/test/te
products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/test/test_hubspot_metadata.py
products/warehouse_sources/backend/temporal/data_imports/sources/hubspot/test/test_hubspot_search.py
products/warehouse_sources/backend/temporal/data_imports/sources/meta_ads/test_meta_ads.py
-products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/tests/test_pipedrive.py
-products/warehouse_sources/backend/temporal/data_imports/sources/postmark/tests/test_postmark.py
+products/warehouse_sources/sources/pipedrive/tests/test_pipedrive.py
+products/warehouse_sources/sources/postmark/tests/test_postmark.py
products/warehouse_sources/backend/temporal/data_imports/sources/sentry/tests/test_sentry.py
products/warehouse_sources/backend/temporal/data_imports/sources/sharepoint/tests/test_sharepoint.py
products/warehouse_sources/backend/temporal/data_imports/sources/trino/tests/test_trino.py
-products/warehouse_sources/backend/temporal/data_imports/sources/webflow/tests/test_webflow.py
-products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/tests/test_zapsign.py
+products/warehouse_sources/sources/webflow/tests/test_webflow.py
+products/warehouse_sources/sources/zapsign/tests/test_zapsign.py
products/warehouse_sources/sources/adjust/tests/test_adjust.py
products/warehouse_sources/sources/ahrefs/tests/test_ahrefs.py
products/warehouse_sources/sources/argocd/tests/test_argocd.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/github/tests/test_github_webhook_template.py b/products/warehouse_sources/backend/temporal/data_imports/sources/github/tests/test_github_webhook_template.py
index 4359461f52e2..55dd23b84c71 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/github/tests/test_github_webhook_template.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/github/tests/test_github_webhook_template.py
@@ -3,7 +3,7 @@ from typing import Any
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
+from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateSimpleTest
from products.warehouse_sources.backend.temporal.data_imports.sources.github.webhook_template import template
@@ -47,7 +47,7 @@ def _status_body(**overrides: Any) -> dict[str, Any]:
return body
-class TestGithubWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestGithubWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_webhook_template.py b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_webhook_template.py
index 4a54951fd67c..0cb5b4c79daf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_webhook_template.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/stripe/tests/test_webhook_template.py
@@ -5,11 +5,11 @@ import hashlib
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
+from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateSimpleTest
from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.webhook_template import template
-class TestStripeWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestStripeWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/sources/attentive/__init__.py b/products/warehouse_sources/sources/attentive/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/attentive.py b/products/warehouse_sources/sources/attentive/_config.py
similarity index 71%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/attentive.py
rename to products/warehouse_sources/sources/attentive/_config.py
index 91b2f16e7fb6..182003b2d82a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/attentive.py
+++ b/products/warehouse_sources/sources/attentive/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/api_client.py b/products/warehouse_sources/sources/attentive/api_client.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/attentive/api_client.py
rename to products/warehouse_sources/sources/attentive/api_client.py
index cdb216a5cbcc..d3037a89c81f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/api_client.py
+++ b/products/warehouse_sources/sources/attentive/api_client.py
@@ -10,18 +10,18 @@ from typing import Any
import requests
import structlog
-from products.warehouse_sources.backend.temporal.data_imports.sources.attentive.constants import (
+from products.warehouse_sources.sources.attentive.constants import (
ATTENTIVE_API_ORIGIN,
ATTENTIVE_V1_BASE_URL,
RESOURCE_TO_ATTENTIVE_EVENT_TYPE,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
ExternalWebhookInfo,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSyncResult,
+ make_tracked_session,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
LOGGER = structlog.get_logger(__name__)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/canonical_descriptions.py b/products/warehouse_sources/sources/attentive/canonical_descriptions.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/attentive/canonical_descriptions.py
rename to products/warehouse_sources/sources/attentive/canonical_descriptions.py
index 702149f16016..9a0f7f836d8a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/attentive/canonical_descriptions.py
@@ -6,9 +6,7 @@ webhook-backed schema names in `constants.py` `RESOURCE_TO_ATTENTIVE_EVENT_TYPE`
`ExternalDataSchema.name` of a synced Attentive table. Columns absent here fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
# Fields present on every stored Attentive webhook row. `event_id` and `created_at` are synthesized
# by PostHog during ingestion; `type` and `timestamp` come from the Attentive webhook payload.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/constants.py b/products/warehouse_sources/sources/attentive/constants.py
similarity index 100%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/attentive/constants.py
rename to products/warehouse_sources/sources/attentive/constants.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/source.py b/products/warehouse_sources/sources/attentive/source.py
similarity index 85%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/attentive/source.py
rename to products/warehouse_sources/sources/attentive/source.py
index e20a902951ef..c020669bcd36 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/source.py
+++ b/products/warehouse_sources/sources/attentive/source.py
@@ -6,43 +6,38 @@ import orjson
import pyarrow as pa
from asgiref.sync import async_to_sync
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.attentive import api_client
-from products.warehouse_sources.backend.temporal.data_imports.sources.attentive.constants import (
+from products.warehouse_sources.sources.attentive import api_client
+from products.warehouse_sources.sources.attentive._config import AttentiveSourceConfig
+from products.warehouse_sources.sources.attentive.constants import (
ATTENTIVE_WEBHOOK_SCHEMA_NAMES,
RESOURCE_TO_ATTENTIVE_EVENT_TYPE,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
ExternalWebhookInfo,
FieldType,
+ ReleaseStatus,
SimpleSource,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
WebhookSyncResult,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.attentive import (
- AttentiveSourceConfig,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
def _webhook_table_transformer(table: pa.Table) -> pa.Table:
@@ -89,7 +84,7 @@ class AttentiveSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.attentive.webhook_template import template
+ from products.warehouse_sources.sources.attentive.webhook_template import template
return template
@@ -160,9 +155,7 @@ class AttentiveSource(
return api_client.validate_credentials(config.api_key)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.attentive.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.attentive.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/attentive/tests/__init__.py b/products/warehouse_sources/sources/attentive/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/tests/test_attentive_api_client.py b/products/warehouse_sources/sources/attentive/tests/test_attentive_api_client.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/attentive/tests/test_attentive_api_client.py
rename to products/warehouse_sources/sources/attentive/tests/test_attentive_api_client.py
index d7012cf8e396..5230d9bc9d50 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/tests/test_attentive_api_client.py
+++ b/products/warehouse_sources/sources/attentive/tests/test_attentive_api_client.py
@@ -1,10 +1,7 @@
import pytest
-from products.warehouse_sources.backend.temporal.data_imports.sources.attentive import api_client
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- scripted_network,
-)
+from products.warehouse_sources.sources.attentive import api_client
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, scripted_network
URL = "https://ph.example/webhook"
WEBHOOKS = "/v1/webhooks"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/tests/test_attentive_source.py b/products/warehouse_sources/sources/attentive/tests/test_attentive_source.py
similarity index 77%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/attentive/tests/test_attentive_source.py
rename to products/warehouse_sources/sources/attentive/tests/test_attentive_source.py
index 64138bcbf806..c290b4848c9f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/tests/test_attentive_source.py
+++ b/products/warehouse_sources/sources/attentive/tests/test_attentive_source.py
@@ -3,17 +3,10 @@ from typing import Any, cast
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.attentive.constants import (
- RESOURCE_TO_ATTENTIVE_EVENT_TYPE,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.attentive.source import (
- AttentiveSource,
- _webhook_table_transformer,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.attentive import (
- AttentiveSourceConfig,
-)
+from products.warehouse_sources.sources.attentive._config import AttentiveSourceConfig
+from products.warehouse_sources.sources.attentive.constants import RESOURCE_TO_ATTENTIVE_EVENT_TYPE
+from products.warehouse_sources.sources.attentive.source import AttentiveSource, _webhook_table_transformer
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
class TestWebhookTableTransformer:
@@ -62,9 +55,7 @@ class TestAttentiveSource:
events = self.source.get_desired_webhook_events(self.config, ["sms_sent", "email_opened", "not_a_schema"])
assert events == ["sms.sent", "email.opened"]
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.attentive.source.api_client.create_webhook"
- )
+ @mock.patch("products.warehouse_sources.sources.attentive.source.api_client.create_webhook")
def test_create_webhook_subscribes_all_event_types(self, mock_create):
self.source.create_webhook(self.config, "https://ph.example/webhook", self.team_id)
@@ -74,9 +65,7 @@ class TestAttentiveSource:
resource_names=list(RESOURCE_TO_ATTENTIVE_EVENT_TYPE.keys()),
)
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.attentive.source.api_client.enable_webhook"
- )
+ @mock.patch("products.warehouse_sources.sources.attentive.source.api_client.enable_webhook")
def test_webhook_inputs_updated_enables_once_secret_provided(self, mock_enable):
mock_enable.return_value = (True, None)
@@ -87,16 +76,14 @@ class TestAttentiveSource:
assert ok is True
mock_enable.assert_called_once_with("key", "https://ph.example/webhook")
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.attentive.source.api_client.enable_webhook"
- )
+ @mock.patch("products.warehouse_sources.sources.attentive.source.api_client.enable_webhook")
def test_webhook_inputs_updated_noop_without_secret(self, mock_enable):
ok, _ = self.source.webhook_inputs_updated(self.config, "https://ph.example/webhook", self.team_id, {})
assert ok is True
mock_enable.assert_not_called()
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.attentive.source.async_to_sync")
+ @mock.patch("products.warehouse_sources.sources.attentive.source.async_to_sync")
def test_source_for_pipeline_metadata(self, mock_async_to_sync):
mock_async_to_sync.return_value = mock.MagicMock(return_value=False)
inputs = mock.MagicMock()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/tests/test_attentive_webhook_template.py b/products/warehouse_sources/sources/attentive/tests/test_attentive_webhook_template.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/attentive/tests/test_attentive_webhook_template.py
rename to products/warehouse_sources/sources/attentive/tests/test_attentive_webhook_template.py
index f6eee61cabb8..c751cfe07d96 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/tests/test_attentive_webhook_template.py
+++ b/products/warehouse_sources/sources/attentive/tests/test_attentive_webhook_template.py
@@ -2,13 +2,13 @@ import hmac
import json
import hashlib
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.attentive.webhook_template import template
+from products.warehouse_sources.sources.attentive.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
SOURCE_ID = "source_test_123"
-class TestAttentiveWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestAttentiveWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/webhook_template.py b/products/warehouse_sources/sources/attentive/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/attentive/webhook_template.py
rename to products/warehouse_sources/sources/attentive/webhook_template.py
index 193c8cd757b9..216b735a63f2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/attentive/webhook_template.py
+++ b/products/warehouse_sources/sources/attentive/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/sources/browse_ai/__init__.py b/products/warehouse_sources/sources/browse_ai/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/browseai.py b/products/warehouse_sources/sources/browse_ai/_config.py
similarity index 71%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/browseai.py
rename to products/warehouse_sources/sources/browse_ai/_config.py
index 45aeb6e1bb17..40329de8cc93 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/browseai.py
+++ b/products/warehouse_sources/sources/browse_ai/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/browse_ai.py b/products/warehouse_sources/sources/browse_ai/browse_ai.py
similarity index 86%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/browse_ai.py
rename to products/warehouse_sources/sources/browse_ai/browse_ai.py
index abe4b0ccf708..12eb968ac3ec 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/browse_ai.py
+++ b/products/warehouse_sources/sources/browse_ai/browse_ai.py
@@ -9,42 +9,36 @@ from asgiref.sync import async_to_sync
from requests import Response
from requests.exceptions import HTTPError
-from posthog.dataclasses import frozen
-
-from products.warehouse_sources.backend.temporal.data_imports.sources.browse_ai.settings import (
+from products.warehouse_sources.sources.browse_ai.settings import (
AUTH_ERROR,
BASE_URL,
ENDPOINTS,
PERMISSION_ERROR,
BrowseAIEndpoint,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
+ BearerTokenAuth,
+ ClientConfig,
+ Endpoint,
ExternalWebhookInfo,
+ PageNumberPaginator,
+ RESTClient,
+ ResumableSourceManager,
+ SinglePagePaginator,
+ SourceInputs,
+ SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import rest_api_resource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import BearerTokenAuth
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
build_dependent_resource,
+ frozen,
+ make_tracked_session,
+ rest_api_resource,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- PageNumberPaginator,
- SinglePagePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- ClientConfig,
- Endpoint,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
if TYPE_CHECKING:
import pyarrow as pa
- from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
+ from products.warehouse_sources.sources.sdk import WebhookSourceManager
@frozen
@@ -171,7 +165,7 @@ def browse_ai_source(
def webhook_table(table: "pa.Table") -> "pa.Table":
- from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import ( # noqa: PLC0415 - keep Arrow conversion off the source registry import path
+ from products.warehouse_sources.sources.sdk.internals import ( # noqa: PLC0415 - keep Arrow conversion off the source registry import path
table_from_py_list,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/canonical_descriptions.py b/products/warehouse_sources/sources/browse_ai/canonical_descriptions.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/canonical_descriptions.py
rename to products/warehouse_sources/sources/browse_ai/canonical_descriptions.py
index 183c94e707d7..febec57fada4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/browse_ai/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"robots": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/settings.py b/products/warehouse_sources/sources/browse_ai/settings.py
similarity index 89%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/settings.py
rename to products/warehouse_sources/sources/browse_ai/settings.py
index dcf0ad2287eb..decf1f90c89d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/settings.py
+++ b/products/warehouse_sources/sources/browse_ai/settings.py
@@ -1,12 +1,7 @@
from dataclasses import field
from typing import Any
-from posthog.dataclasses import frozen
-
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
- DependentEndpointConfig,
-)
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.sources.sdk import DependentEndpointConfig, IncrementalField, frozen
BASE_URL = "https://api.browse.ai/v2"
AUTH_ERROR = "Your Browse AI API key is invalid or expired. Generate a new key and reconnect."
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/source.py b/products/warehouse_sources/sources/browse_ai/source.py
similarity index 82%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/source.py
rename to products/warehouse_sources/sources/browse_ai/source.py
index 68379bdf6272..d59d2acfb991 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/source.py
+++ b/products/warehouse_sources/sources/browse_ai/source.py
@@ -1,49 +1,42 @@
from typing import cast
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.browse_ai.browse_ai import (
+from products.warehouse_sources.sources.browse_ai._config import BrowseAISourceConfig
+from products.warehouse_sources.sources.browse_ai.browse_ai import (
BrowseAIResumeConfig,
BrowseAIWebhooks,
browse_ai_source,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.browse_ai.settings import (
+from products.warehouse_sources.sources.browse_ai.settings import (
AUTH_ERROR,
ENDPOINTS,
INCREMENTAL_FIELDS,
PERMISSION_ERROR,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
ExternalWebhookInfo,
FieldType,
+ HogFunctionTemplateDC,
+ ReleaseStatus,
ResumableSource,
+ ResumableSourceManager,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
+ WebhookSourceManager,
build_endpoint_schemas,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.browseai import (
- BrowseAISourceConfig,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
@SourceRegistry.register
@@ -112,7 +105,7 @@ class BrowseAISource(ResumableSource[BrowseAISourceConfig, BrowseAIResumeConfig]
return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, supports_webhooks={"tasks"})
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.browse_ai.canonical_descriptions import ( # noqa: PLC0415 - load descriptions only when enrichment needs them
+ from products.warehouse_sources.sources.browse_ai.canonical_descriptions import ( # noqa: PLC0415 - load descriptions only when enrichment needs them
CANONICAL_DESCRIPTIONS,
)
diff --git a/products/warehouse_sources/sources/browse_ai/tests/__init__.py b/products/warehouse_sources/sources/browse_ai/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/tests/test_browse_ai.py b/products/warehouse_sources/sources/browse_ai/tests/test_browse_ai.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/tests/test_browse_ai.py
rename to products/warehouse_sources/sources/browse_ai/tests/test_browse_ai.py
index b0fe929d4321..8841e5db8735 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/tests/test_browse_ai.py
+++ b/products/warehouse_sources/sources/browse_ai/tests/test_browse_ai.py
@@ -9,18 +9,14 @@ import pyarrow as pa
import requests_mock
from requests.exceptions import HTTPError
-from products.warehouse_sources.backend.temporal.data_imports.sources.browse_ai.browse_ai import (
+from products.warehouse_sources.sources.browse_ai.browse_ai import (
BrowseAIResumeConfig,
browse_ai_source,
validate_credentials,
webhook_table,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.browse_ai.settings import BASE_URL
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
- RESTClient,
- RESTClientRetryableError,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
+from products.warehouse_sources.sources.browse_ai.settings import BASE_URL
+from products.warehouse_sources.sources.sdk import RESTClient, RESTClientRetryableError, SourceInputs, SourceResponse
def sync_items(response: SourceResponse) -> Iterable[Any]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/tests/test_browse_ai_source.py b/products/warehouse_sources/sources/browse_ai/tests/test_browse_ai_source.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/tests/test_browse_ai_source.py
rename to products/warehouse_sources/sources/browse_ai/tests/test_browse_ai_source.py
index a6e3218805e4..164d78564439 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/browse_ai/tests/test_browse_ai_source.py
+++ b/products/warehouse_sources/sources/browse_ai/tests/test_browse_ai_source.py
@@ -6,14 +6,10 @@ from unittest.mock import Mock
import requests_mock
-from products.cdp.backend.validation import compile_hog
-from products.warehouse_sources.backend.temporal.data_imports.sources.browse_ai.settings import BASE_URL
-from products.warehouse_sources.backend.temporal.data_imports.sources.browse_ai.source import BrowseAISource
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.browseai import (
- BrowseAISourceConfig,
-)
-
-from common.hogvm.python.execute import execute_bytecode
+from products.warehouse_sources.sources.browse_ai._config import BrowseAISourceConfig
+from products.warehouse_sources.sources.browse_ai.settings import BASE_URL
+from products.warehouse_sources.sources.browse_ai.source import BrowseAISource
+from products.warehouse_sources.sources.sdk.testing import compile_hog, execute_bytecode
ROBOT = "00000000-0000-4000-8000-000000000001"
SECOND_ROBOT = "00000000-0000-4000-8000-000000000002"
diff --git a/products/warehouse_sources/sources/calendly/__init__.py b/products/warehouse_sources/sources/calendly/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/calendly.py b/products/warehouse_sources/sources/calendly/_config.py
similarity index 72%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/calendly.py
rename to products/warehouse_sources/sources/calendly/_config.py
index 04c828b383a1..7b93795c7c9f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/calendly.py
+++ b/products/warehouse_sources/sources/calendly/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/api_inventory.md b/products/warehouse_sources/sources/calendly/api_inventory.md
similarity index 100%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/calendly/api_inventory.md
rename to products/warehouse_sources/sources/calendly/api_inventory.md
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/calendly.py b/products/warehouse_sources/sources/calendly/calendly.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/calendly/calendly.py
rename to products/warehouse_sources/sources/calendly/calendly.py
index ff5b713059d9..78963e19ce9d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/calendly.py
+++ b/products/warehouse_sources/sources/calendly/calendly.py
@@ -10,8 +10,7 @@ import pyarrow as pa
from asgiref.sync import async_to_sync
from requests import Session
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.settings import (
+from products.warehouse_sources.sources.calendly.settings import (
CALENDLY_ENDPOINTS,
CALENDLY_WEBHOOK_EVENTS,
PAGE_SIZE,
@@ -20,30 +19,23 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.s
WEBHOOK_SCHEMA_NAMES,
CalendlyEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
+ ClientConfig,
+ Endpoint,
ExternalWebhookInfo,
+ JSONResponsePaginator,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
+ WebhookSourceManager,
build_dependent_resource,
+ make_tracked_session,
+ rest_api_resource,
+ validate_via_probe,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- JSONResponsePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- ClientConfig,
- Endpoint,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
CALENDLY_BASE_URL = "https://api.calendly.com"
REQUEST_TIMEOUT = 60
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/canonical_descriptions.py b/products/warehouse_sources/sources/calendly/canonical_descriptions.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/calendly/canonical_descriptions.py
rename to products/warehouse_sources/sources/calendly/canonical_descriptions.py
index 7739af54c311..ce9b5830b01b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/calendly/canonical_descriptions.py
@@ -5,9 +5,7 @@ Keyed by the endpoint names in `settings.py` `CALENDLY_ENDPOINTS`, which match t
`ExternalDataSchema.name` of a synced Calendly table. Columns absent here fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
# Fields shared by most Calendly resources; merged into each entry so we don't repeat them.
_COMMON_COLUMNS = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/settings.py b/products/warehouse_sources/sources/calendly/settings.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/calendly/settings.py
rename to products/warehouse_sources/sources/calendly/settings.py
index 948163d63bc2..d1ad542f357c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/settings.py
+++ b/products/warehouse_sources/sources/calendly/settings.py
@@ -1,10 +1,7 @@
from dataclasses import dataclass, field
from typing import Optional
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
- DependentEndpointConfig,
-)
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import DependentEndpointConfig, IncrementalField, IncrementalFieldType
PAGE_SIZE = 100
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/source.py b/products/warehouse_sources/sources/calendly/source.py
similarity index 85%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/calendly/source.py
rename to products/warehouse_sources/sources/calendly/source.py
index 6f4bee1bb5ea..0d1d1f81d5c2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/source.py
+++ b/products/warehouse_sources/sources/calendly/source.py
@@ -1,14 +1,7 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.calendly import (
+from products.warehouse_sources.sources.calendly._config import CalendlySourceConfig
+from products.warehouse_sources.sources.calendly.calendly import (
CALENDLY_API_VERSION_V1,
CALENDLY_API_VERSION_V2,
SUPPORTED_API_VERSIONS,
@@ -19,40 +12,40 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.c
get_external_webhook_info as get_calendly_webhook_info,
validate_credentials as validate_calendly_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.settings import (
+from products.warehouse_sources.sources.calendly.settings import (
CALENDLY_WEBHOOK_EVENTS,
ENDPOINTS,
INCREMENTAL_FIELDS,
WEBHOOK_RESOURCE_MAP,
WEBHOOK_SCHEMA_NAMES,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
ExternalWebhookInfo,
FieldType,
+ ReleaseStatus,
ResumableSource,
+ ResumableSourceManager,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
VersionDeprecation,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
+ WebhookSourceManager,
build_endpoint_schemas,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.calendly import (
- CalendlySourceConfig,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@SourceRegistry.register
@@ -79,7 +72,7 @@ class CalendlySource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.webhook_template import template
+ from products.warehouse_sources.sources.calendly.webhook_template import template
return template
@@ -146,9 +139,7 @@ Paste the same signing key into the field below so PostHog can verify deliveries
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.calendly.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/calendly/tests/__init__.py b/products/warehouse_sources/sources/calendly/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/tests/test_calendly.py b/products/warehouse_sources/sources/calendly/tests/test_calendly.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/calendly/tests/test_calendly.py
rename to products/warehouse_sources/sources/calendly/tests/test_calendly.py
index 9c92c60139af..bde4a85e1fdb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/tests/test_calendly.py
+++ b/products/warehouse_sources/sources/calendly/tests/test_calendly.py
@@ -7,8 +7,8 @@ from unittest import mock
from parameterized import parameterized
from requests import HTTPError, Response
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.calendly import (
+from products.warehouse_sources.sources.calendly._config import CalendlySourceConfig
+from products.warehouse_sources.sources.calendly.calendly import (
CALENDLY_BASE_URL,
SUPPORTED_API_VERSIONS,
CalendlyResumeConfig,
@@ -20,27 +20,20 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.c
get_external_webhook_info,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.settings import (
- CALENDLY_WEBHOOK_EVENTS,
- ENDPOINTS,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.source import CalendlySource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+from products.warehouse_sources.sources.calendly.settings import CALENDLY_WEBHOOK_EVENTS, ENDPOINTS
+from products.warehouse_sources.sources.calendly.source import CalendlySource
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import (
RecordedRequest,
ScriptedResponse,
SourceDriver,
always,
scripted_network,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.calendly import (
- CalendlySourceConfig,
-)
# The two webhook-manager integration tests still need direct session patches.
CLIENT_SESSION_PATCH = "products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client.make_tracked_session"
-CALENDLY_SESSION_PATCH = (
- "products.warehouse_sources.backend.temporal.data_imports.sources.calendly.calendly.make_tracked_session"
-)
+CALENDLY_SESSION_PATCH = "products.warehouse_sources.sources.calendly.calendly.make_tracked_session"
ORG_URI = "https://api.calendly.com/organizations/ABC123"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/tests/test_calendly_source.py b/products/warehouse_sources/sources/calendly/tests/test_calendly_source.py
similarity index 85%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/calendly/tests/test_calendly_source.py
rename to products/warehouse_sources/sources/calendly/tests/test_calendly_source.py
index 4aa9404cf630..b26ea2870a9a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/tests/test_calendly_source.py
+++ b/products/warehouse_sources/sources/calendly/tests/test_calendly_source.py
@@ -1,15 +1,10 @@
import pytest
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.calendly import (
- CALENDLY_API_VERSION_V1,
- CALENDLY_API_VERSION_V2,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.settings import CALENDLY_WEBHOOK_EVENTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.source import CalendlySource
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.calendly import (
- CalendlySourceConfig,
-)
+from products.warehouse_sources.sources.calendly._config import CalendlySourceConfig
+from products.warehouse_sources.sources.calendly.calendly import CALENDLY_API_VERSION_V1, CALENDLY_API_VERSION_V2
+from products.warehouse_sources.sources.calendly.settings import CALENDLY_WEBHOOK_EVENTS
+from products.warehouse_sources.sources.calendly.source import CalendlySource
def _make_inputs(schema_name: str = "scheduled_events", **overrides):
@@ -46,9 +41,7 @@ class TestCalendlySource:
((False, None), False, "couldn't reach Calendly"),
],
)
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.calendly.source.validate_calendly_credentials"
- )
+ @mock.patch("products.warehouse_sources.sources.calendly.source.validate_calendly_credentials")
def test_validate_credentials(self, mock_validate, mock_return, expected_valid, expected_message_fragment):
mock_validate.return_value = mock_return
@@ -62,7 +55,7 @@ class TestCalendlySource:
assert expected_message_fragment in error_message
mock_validate.assert_called_once_with(self.config.personal_access_token)
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.calendly.source.calendly_source")
+ @mock.patch("products.warehouse_sources.sources.calendly.source.calendly_source")
def test_source_for_pipeline_plumbs_arguments(self, mock_calendly_source):
inputs = _make_inputs(
schema_name="scheduled_events",
@@ -86,7 +79,7 @@ class TestCalendlySource:
assert self.source.default_version == CALENDLY_API_VERSION_V2
assert set(self.source.supported_versions) == {CALENDLY_API_VERSION_V1, CALENDLY_API_VERSION_V2}
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.calendly.source.calendly_source")
+ @mock.patch("products.warehouse_sources.sources.calendly.source.calendly_source")
def test_source_for_pipeline_drops_last_value_when_not_incremental(self, mock_calendly_source):
inputs = _make_inputs(
should_use_incremental_field=False,
@@ -99,7 +92,7 @@ class TestCalendlySource:
WEBHOOK_URL = "https://webhooks.us.posthog.com/public/webhooks/dwh/hog-fn-1"
-WEBHOOK_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.calendly.source"
+WEBHOOK_MODULE = "products.warehouse_sources.sources.calendly.source"
class TestCalendlySourceWebhooks:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/tests/test_calendly_webhook_template.py b/products/warehouse_sources/sources/calendly/tests/test_calendly_webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/calendly/tests/test_calendly_webhook_template.py
rename to products/warehouse_sources/sources/calendly/tests/test_calendly_webhook_template.py
index 814bcf0ef31a..ff15f9cdcc27 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/tests/test_calendly_webhook_template.py
+++ b/products/warehouse_sources/sources/calendly/tests/test_calendly_webhook_template.py
@@ -5,8 +5,8 @@ import hashlib
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.calendly.webhook_template import template
+from products.warehouse_sources.sources.calendly.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
SOURCE_ID = "source_test_123"
SIGNING_KEY = "cal_signing_key_test"
@@ -35,7 +35,7 @@ INVITEE_CREATED = {
}
-class TestCalendlyWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestCalendlyWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/webhook_template.py b/products/warehouse_sources/sources/calendly/webhook_template.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/calendly/webhook_template.py
rename to products/warehouse_sources/sources/calendly/webhook_template.py
index 1b78f5a85b70..4aff4273c6e8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/calendly/webhook_template.py
+++ b/products/warehouse_sources/sources/calendly/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/sources/chatwoot/__init__.py b/products/warehouse_sources/sources/chatwoot/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/chatwoot.py b/products/warehouse_sources/sources/chatwoot/_config.py
similarity index 76%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/chatwoot.py
rename to products/warehouse_sources/sources/chatwoot/_config.py
index 54e66d18157e..5e11535abf31 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/chatwoot.py
+++ b/products/warehouse_sources/sources/chatwoot/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/canonical_descriptions.py b/products/warehouse_sources/sources/chatwoot/canonical_descriptions.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/canonical_descriptions.py
rename to products/warehouse_sources/sources/chatwoot/canonical_descriptions.py
index 417656d38171..5703a12c8549 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/chatwoot/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"conversations": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/chatwoot.py b/products/warehouse_sources/sources/chatwoot/chatwoot.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/chatwoot.py
rename to products/warehouse_sources/sources/chatwoot/chatwoot.py
index fa54875f0287..e1fee0d4b00c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/chatwoot.py
+++ b/products/warehouse_sources/sources/chatwoot/chatwoot.py
@@ -14,10 +14,7 @@ from asgiref.sync import async_to_sync
from structlog.types import FilteringBoundLogger
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from posthog.dataclasses import frozen
-
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.settings import (
+from products.warehouse_sources.sources.chatwoot.settings import (
CHATWOOT_ENDPOINTS,
DEFAULT_HOST,
MESSAGE_TYPE_TO_INT,
@@ -25,18 +22,20 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.s
ChatwootEndpointConfig,
all_webhook_events,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
ExternalWebhookInfo,
+ ResumableSourceManager,
+ SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
+ WebhookSourceManager,
WebhookSyncResult,
+ _is_host_safe,
+ frozen,
+ make_tracked_session,
+ parse_datetime_value,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import parse_datetime_value
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import _is_host_safe
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
REQUEST_TIMEOUT_SECONDS = 60
MAX_RETRIES = 5
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/settings.py b/products/warehouse_sources/sources/chatwoot/settings.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/settings.py
rename to products/warehouse_sources/sources/chatwoot/settings.py
index 0c4aa95e79e2..af5beea54036 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/settings.py
+++ b/products/warehouse_sources/sources/chatwoot/settings.py
@@ -1,8 +1,7 @@
from dataclasses import dataclass, field
from typing import Literal, Optional
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SortMode
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType, SortMode
DEFAULT_HOST = "https://app.chatwoot.com"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/source.py b/products/warehouse_sources/sources/chatwoot/source.py
similarity index 88%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/source.py
rename to products/warehouse_sources/sources/chatwoot/source.py
index acd0325b990d..10b3aa164753 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/source.py
+++ b/products/warehouse_sources/sources/chatwoot/source.py
@@ -2,19 +2,13 @@ from typing import TYPE_CHECKING, Optional, cast
import structlog
-from products.warehouse_sources.backend.facade.codebase_detection import detect
+from products.warehouse_sources.sources.sdk import detect
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.chatwoot import (
+from products.warehouse_sources.sources.chatwoot._config import ChatwootSourceConfig
+from products.warehouse_sources.sources.chatwoot.chatwoot import (
AUDIT_LOGS_UNAVAILABLE_ERROR,
HOST_NOT_ALLOWED_ERROR,
HTTP_NOT_ALLOWED_ERROR,
@@ -30,33 +24,34 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.c
update_webhook_events as update_chatwoot_webhook_events,
validate_credentials as validate_chatwoot_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.settings import (
+from products.warehouse_sources.sources.chatwoot.settings import (
CHATWOOT_ENDPOINTS,
ENDPOINTS,
RESOURCE_TO_WEBHOOK_OBJECT_TYPE,
all_webhook_events,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
ExternalWebhookInfo,
FieldType,
+ ReleaseStatus,
ResumableSource,
+ ResumableSourceManager,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
WebhookSyncResult,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.chatwoot import (
- ChatwootSourceConfig,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
# Webhook management calls run in the API request path where no job logger exists.
logger = structlog.get_logger(__name__)
@@ -158,7 +153,7 @@ If automatic creation failed, note that only Chatwoot administrators can manage
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.canonical_descriptions import ( # noqa: PLC0415
+ from products.warehouse_sources.sources.chatwoot.canonical_descriptions import ( # noqa: PLC0415
CANONICAL_DESCRIPTIONS,
)
@@ -166,9 +161,7 @@ If automatic creation failed, note that only Chatwoot administrators can manage
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.webhook_template import ( # noqa: PLC0415
- template,
- )
+ from products.warehouse_sources.sources.chatwoot.webhook_template import template # noqa: PLC0415
return template
diff --git a/products/warehouse_sources/sources/chatwoot/tests/__init__.py b/products/warehouse_sources/sources/chatwoot/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/tests/test_chatwoot.py b/products/warehouse_sources/sources/chatwoot/tests/test_chatwoot.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/tests/test_chatwoot.py
rename to products/warehouse_sources/sources/chatwoot/tests/test_chatwoot.py
index 9cbdf0200488..e9176c9edf3b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/tests/test_chatwoot.py
+++ b/products/warehouse_sources/sources/chatwoot/tests/test_chatwoot.py
@@ -7,8 +7,8 @@ from unittest import mock
import requests
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.chatwoot import (
+from products.warehouse_sources.sources.chatwoot._config import ChatwootSourceConfig
+from products.warehouse_sources.sources.chatwoot.chatwoot import (
AUDIT_LOGS_UNAVAILABLE_ERROR,
REPORTING_EVENTS_UNAVAILABLE_ERROR,
ChatwootResumeConfig,
@@ -21,21 +21,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.c
normalize_host,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.settings import (
- CHATWOOT_ENDPOINTS,
- ENDPOINTS,
- MESSAGES_PAGE_SIZE,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.source import ChatwootSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
- always,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.chatwoot import (
- ChatwootSourceConfig,
-)
+from products.warehouse_sources.sources.chatwoot.settings import CHATWOOT_ENDPOINTS, ENDPOINTS, MESSAGES_PAGE_SIZE
+from products.warehouse_sources.sources.chatwoot.source import ChatwootSource
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network
TEAM_ID = 1
@@ -200,9 +189,7 @@ class TestGetRowsPaged:
class TestGetRowsMessages:
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.chatwoot.make_tracked_session"
- )
+ @mock.patch("products.warehouse_sources.sources.chatwoot.chatwoot.make_tracked_session")
def test_deleted_conversation_404_is_skipped(self, mock_session):
mock_session.return_value.get.side_effect = [
_resp(_conversations_page([{"id": 1}, {"id": 2}])),
@@ -217,12 +204,10 @@ class TestGetRowsMessages:
assert [item["id"] for batch in batches for item in batch] == [400]
@mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.chatwoot.MAX_MESSAGE_PAGES_PER_SYNC",
+ "products.warehouse_sources.sources.chatwoot.chatwoot.MAX_MESSAGE_PAGES_PER_SYNC",
2,
)
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.chatwoot.make_tracked_session"
- )
+ @mock.patch("products.warehouse_sources.sources.chatwoot.chatwoot.make_tracked_session")
def test_message_walk_stops_at_the_aggregate_page_cap(self, mock_session):
# A hostile server can keep returning full, advancing pages; the per-conversation cap alone
# would still let conversation count multiply requests without bound, so the walk must stop
@@ -251,9 +236,7 @@ class TestGetRowsMessages:
assert len(message_urls) == 2
assert not any("/conversations/2/messages" in url for url in message_urls)
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.chatwoot.make_tracked_session"
- )
+ @mock.patch("products.warehouse_sources.sources.chatwoot.chatwoot.make_tracked_session")
def test_resumes_from_bookmarked_conversation_and_cursor(self, mock_session):
mock_session.return_value.get.side_effect = [
_resp(_conversations_page([{"id": 1}, {"id": 2}])),
@@ -308,7 +291,7 @@ class TestGetRowsMemberFanout:
assert result.paths[1:] == [f"/api/v1/accounts/1/teams/{team_id}/team_members" for team_id in expected_team_ids]
@mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.chatwoot.MAX_MEMBER_FANOUT_PARENTS",
+ "products.warehouse_sources.sources.chatwoot.chatwoot.MAX_MEMBER_FANOUT_PARENTS",
1,
)
def test_parent_fanout_stops_at_the_parent_cap(self):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/tests/test_chatwoot_source.py b/products/warehouse_sources/sources/chatwoot/tests/test_chatwoot_source.py
similarity index 85%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/tests/test_chatwoot_source.py
rename to products/warehouse_sources/sources/chatwoot/tests/test_chatwoot_source.py
index 2888c0f5a7a8..089953da28f8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/tests/test_chatwoot_source.py
+++ b/products/warehouse_sources/sources/chatwoot/tests/test_chatwoot_source.py
@@ -1,9 +1,7 @@
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.source import ChatwootSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.chatwoot import (
- ChatwootSourceConfig,
-)
+from products.warehouse_sources.sources.chatwoot._config import ChatwootSourceConfig
+from products.warehouse_sources.sources.chatwoot.source import ChatwootSource
class TestChatwootSource:
@@ -30,9 +28,7 @@ class TestChatwootSource:
input_keys = {schema_input["key"] for schema_input in template.inputs_schema or []}
assert {"signing_secret", "bypass_signature_check", "schema_mapping", "source_id"} <= input_keys
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.source.create_chatwoot_webhook"
- )
+ @mock.patch("products.warehouse_sources.sources.chatwoot.source.create_chatwoot_webhook")
def test_create_webhook_plumbs_config(self, mock_create):
self.source.create_webhook(self.config, "https://ph/webhook", self.team_id)
@@ -57,7 +53,7 @@ class TestChatwootSource:
"message_updated",
}
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.chatwoot.source.chatwoot_source")
+ @mock.patch("products.warehouse_sources.sources.chatwoot.source.chatwoot_source")
def test_source_for_pipeline_plumbs_arguments(self, mock_chatwoot_source):
inputs = mock.MagicMock()
inputs.schema_name = "conversations"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/webhook_template.py b/products/warehouse_sources/sources/chatwoot/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/webhook_template.py
rename to products/warehouse_sources/sources/chatwoot/webhook_template.py
index dfe3311b007b..47da5031044b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/chatwoot/webhook_template.py
+++ b/products/warehouse_sources/sources/chatwoot/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
# Chatwoot signs deliveries (recent versions) with X-Chatwoot-Signature: sha256=<hex hmac-sha256 of
# "<X-Chatwoot-Timestamp>.<raw body>">. Older self-hosted installs have no per-webhook secret and
diff --git a/products/warehouse_sources/sources/clerk/__init__.py b/products/warehouse_sources/sources/clerk/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/clerk.py b/products/warehouse_sources/sources/clerk/_config.py
similarity index 71%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/clerk.py
rename to products/warehouse_sources/sources/clerk/_config.py
index 2dd8bd692b0c..f0e2fcaec01c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/clerk.py
+++ b/products/warehouse_sources/sources/clerk/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/canonical_descriptions.py b/products/warehouse_sources/sources/clerk/canonical_descriptions.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/clerk/canonical_descriptions.py
rename to products/warehouse_sources/sources/clerk/canonical_descriptions.py
index ffb41d725bc7..d2693768d701 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/clerk/canonical_descriptions.py
@@ -5,9 +5,7 @@ Keyed by the endpoint names in `settings.py` `CLERK_ENDPOINTS`, which match the
`ExternalDataSchema.name` of a synced Clerk table. Columns absent here fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
# Fields shared by most Clerk objects; merged into each entry so we don't repeat them.
_COMMON_COLUMNS = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py b/products/warehouse_sources/sources/clerk/clerk.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py
rename to products/warehouse_sources/sources/clerk/clerk.py
index 312dc9bedfe9..011b7e1081de 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/clerk.py
+++ b/products/warehouse_sources/sources/clerk/clerk.py
@@ -11,28 +11,25 @@ from requests import Request, Response
from requests.exceptions import HTTPError, RequestException
from structlog.types import FilteringBoundLogger
-from posthog.exceptions_capture import capture_exception
-
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import (
+from products.warehouse_sources.sources.clerk.settings import (
CLERK_ENDPOINTS,
RETIRED_ENDPOINTS,
WEBHOOK_SCHEMA_NAMES,
ClerkEndpointConfig,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resources,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
+from products.warehouse_sources.sources.sdk import (
+ BasePaginator,
Endpoint,
EndpointResource,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SourceResponse,
+ WebhookSourceManager,
+ capture_exception,
+ make_tracked_session,
+ rest_api_resources,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
@dataclasses.dataclass
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py b/products/warehouse_sources/sources/clerk/settings.py
similarity index 100%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/clerk/settings.py
rename to products/warehouse_sources/sources/clerk/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/source.py b/products/warehouse_sources/sources/clerk/source.py
similarity index 90%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/clerk/source.py
rename to products/warehouse_sources/sources/clerk/source.py
index d999530104d8..80c08e4de7ca 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/source.py
+++ b/products/warehouse_sources/sources/clerk/source.py
@@ -1,45 +1,38 @@
from typing import TYPE_CHECKING, Optional, cast
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.clerk import (
+from products.warehouse_sources.sources.clerk._config import ClerkSourceConfig
+from products.warehouse_sources.sources.clerk.clerk import (
FORBIDDEN_KEY_MESSAGE,
INSTANCE_NOT_FOUND_MESSAGE,
ClerkResumeConfig,
clerk_source,
validate_credentials as validate_clerk_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import (
- ENDPOINTS,
- RETIRED_ENDPOINTS,
- WEBHOOK_SCHEMA_NAMES,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.clerk.settings import ENDPOINTS, RETIRED_ENDPOINTS, WEBHOOK_SCHEMA_NAMES
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
FieldType,
+ ReleaseStatus,
ResumableSource,
+ ResumableSourceManager,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.clerk import ClerkSourceConfig
-from products.warehouse_sources.backend.types import ExternalDataSourceType
@SourceRegistry.register
@@ -169,7 +162,7 @@ class ClerkSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.webhook_template import template
+ from products.warehouse_sources.sources.clerk.webhook_template import template
return template
@@ -230,9 +223,7 @@ Clerk delete events only carry the record ID, so webhook sync doesn't remove del
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.clerk.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/clerk/tests/__init__.py b/products/warehouse_sources/sources/clerk/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py b/products/warehouse_sources/sources/clerk/tests/test_clerk.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py
rename to products/warehouse_sources/sources/clerk/tests/test_clerk.py
index 1d65138b2bc6..354d2a7a068f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk.py
+++ b/products/warehouse_sources/sources/clerk/tests/test_clerk.py
@@ -10,7 +10,8 @@ from requests import Response
from requests.exceptions import HTTPError, RequestException
from structlog.testing import capture_logs
-from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.clerk import (
+from products.warehouse_sources.sources.clerk._config import ClerkSourceConfig
+from products.warehouse_sources.sources.clerk.clerk import (
ClerkPaginator,
ClerkResumeConfig,
_convert_timestamps,
@@ -18,18 +19,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.cler
_webhook_table_transformer,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import (
- CLERK_ENDPOINTS,
- RETIRED_ENDPOINTS,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.source import ClerkSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
- always,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.clerk import ClerkSourceConfig
+from products.warehouse_sources.sources.clerk.settings import CLERK_ENDPOINTS, RETIRED_ENDPOINTS
+from products.warehouse_sources.sources.clerk.source import ClerkSource
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network
class TestClerkPaginator:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk_webhook_template.py b/products/warehouse_sources/sources/clerk/tests/test_clerk_webhook_template.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk_webhook_template.py
rename to products/warehouse_sources/sources/clerk/tests/test_clerk_webhook_template.py
index 177e4fef2da1..3e373dfedfd9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/tests/test_clerk_webhook_template.py
+++ b/products/warehouse_sources/sources/clerk/tests/test_clerk_webhook_template.py
@@ -6,9 +6,9 @@ import hashlib
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateSimpleTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.settings import WEBHOOK_EVENT_TO_SCHEMA
-from products.warehouse_sources.backend.temporal.data_imports.sources.clerk.webhook_template import template
+from products.warehouse_sources.sources.clerk.settings import WEBHOOK_EVENT_TO_SCHEMA
+from products.warehouse_sources.sources.clerk.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
# Svix secrets are random bytes. These are not valid UTF-8, so a template that uses the secret as a
# string key fails to verify.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/webhook_template.py b/products/warehouse_sources/sources/clerk/webhook_template.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/clerk/webhook_template.py
rename to products/warehouse_sources/sources/clerk/webhook_template.py
index 931ab307e90b..5360c2827d3e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/clerk/webhook_template.py
+++ b/products/warehouse_sources/sources/clerk/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
# Clerk sends webhooks through Svix. Svix signs "{svix-id}.{svix-timestamp}.{raw body}" with
# HMAC-SHA256. The key is the base64 part of the whsec_ signing secret, decoded to bytes. The
diff --git a/products/warehouse_sources/sources/featurebase/__init__.py b/products/warehouse_sources/sources/featurebase/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/featurebase.py b/products/warehouse_sources/sources/featurebase/_config.py
similarity index 72%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/featurebase.py
rename to products/warehouse_sources/sources/featurebase/_config.py
index ff70246e2d24..8b8de254219f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/featurebase.py
+++ b/products/warehouse_sources/sources/featurebase/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/canonical_descriptions.py b/products/warehouse_sources/sources/featurebase/canonical_descriptions.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/canonical_descriptions.py
rename to products/warehouse_sources/sources/featurebase/canonical_descriptions.py
index ad867ad1983d..14206d76396e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/featurebase/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
# Sourced from the official Featurebase API reference (https://developers.featurebase.app/api).
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/featurebase.py b/products/warehouse_sources/sources/featurebase/featurebase.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/featurebase.py
rename to products/warehouse_sources/sources/featurebase/featurebase.py
index cd8c08dbb78b..9ee16afcfbee 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/featurebase.py
+++ b/products/warehouse_sources/sources/featurebase/featurebase.py
@@ -11,18 +11,7 @@ from asgiref.sync import async_to_sync
from structlog.types import FilteringBoundLogger
from tenacity import retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SortMode, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.featurebase.settings import (
+from products.warehouse_sources.sources.featurebase.settings import (
FEATUREBASE_API_VERSION,
FEATUREBASE_BASE_URL,
FEATUREBASE_ENDPOINTS,
@@ -31,6 +20,18 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.featurebas
FeaturebaseEndpointConfig,
FeaturebaseFanOutConfig,
)
+from products.warehouse_sources.sources.sdk import (
+ ExternalWebhookInfo,
+ ResumableSourceManager,
+ SortMode,
+ SourceResponse,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSourceManager,
+ WebhookSyncResult,
+ make_tracked_session,
+)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
POSTHOG_WEBHOOK_NAME = "PostHog data warehouse"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/settings.py b/products/warehouse_sources/sources/featurebase/settings.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/settings.py
rename to products/warehouse_sources/sources/featurebase/settings.py
index 826f8ce93215..17dead36ee64 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/settings.py
+++ b/products/warehouse_sources/sources/featurebase/settings.py
@@ -1,7 +1,7 @@
from dataclasses import dataclass, field
from typing import Literal, Optional
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType
FEATUREBASE_BASE_URL = "https://do.featurebase.app/v2"
FEATUREBASE_API_VERSION = "2026-01-01.nova"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/source.py b/products/warehouse_sources/sources/featurebase/source.py
similarity index 85%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/source.py
rename to products/warehouse_sources/sources/featurebase/source.py
index da3d92d8ee5b..12fec93e6866 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/source.py
+++ b/products/warehouse_sources/sources/featurebase/source.py
@@ -1,31 +1,7 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSource,
- WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.featurebase.featurebase import (
+from products.warehouse_sources.sources.featurebase._config import FeaturebaseSourceConfig
+from products.warehouse_sources.sources.featurebase.featurebase import (
FeaturebaseResumeConfig,
all_desired_webhook_topics,
create_webhook as create_featurebase_webhook,
@@ -35,19 +11,38 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.featurebas
sync_webhook_events as sync_featurebase_webhook_events,
validate_credentials as validate_featurebase_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.featurebase.settings import (
+from products.warehouse_sources.sources.featurebase.settings import (
ENDPOINTS,
FEATUREBASE_ENDPOINTS,
INCREMENTAL_FIELDS,
RESOURCE_TO_FEATUREBASE_OBJECT_TYPE,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.featurebase import (
- FeaturebaseSourceConfig,
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
+ ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSource,
+ WebhookSourceManager,
+ WebhookSyncResult,
+ detect,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@SourceRegistry.register
@@ -119,9 +114,7 @@ class FeaturebaseSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.featurebase.webhook_template import (
- template,
- )
+ from products.warehouse_sources.sources.featurebase.webhook_template import template
return template
@@ -130,9 +123,7 @@ class FeaturebaseSource(
return RESOURCE_TO_FEATUREBASE_OBJECT_TYPE
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.featurebase.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.featurebase.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/featurebase/tests/__init__.py b/products/warehouse_sources/sources/featurebase/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/tests/test_featurebase.py b/products/warehouse_sources/sources/featurebase/tests/test_featurebase.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/tests/test_featurebase.py
rename to products/warehouse_sources/sources/featurebase/tests/test_featurebase.py
index dfe1e7961ae7..78d669b7297a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/tests/test_featurebase.py
+++ b/products/warehouse_sources/sources/featurebase/tests/test_featurebase.py
@@ -7,13 +7,8 @@ from unittest.mock import MagicMock, patch
import requests
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.featurebase import featurebase
-from products.warehouse_sources.backend.temporal.data_imports.sources.featurebase.featurebase import (
+from products.warehouse_sources.sources.featurebase import featurebase
+from products.warehouse_sources.sources.featurebase.featurebase import (
FeaturebaseResumeConfig,
_build_initial_params,
_page_predates_cutoff,
@@ -26,7 +21,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.featurebas
sync_webhook_events,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.featurebase.settings import FEATUREBASE_ENDPOINTS
+from products.warehouse_sources.sources.featurebase.settings import FEATUREBASE_ENDPOINTS
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, scripted_network
BASE = "https://do.featurebase.app/v2"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/tests/test_featurebase_source.py b/products/warehouse_sources/sources/featurebase/tests/test_featurebase_source.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/tests/test_featurebase_source.py
rename to products/warehouse_sources/sources/featurebase/tests/test_featurebase_source.py
index f1e3a0ce926c..1804a5be52dd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/tests/test_featurebase_source.py
+++ b/products/warehouse_sources/sources/featurebase/tests/test_featurebase_source.py
@@ -4,10 +4,10 @@ from unittest.mock import MagicMock, patch
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
-from products.warehouse_sources.backend.temporal.data_imports.sources.featurebase.source import FeaturebaseSource
+from products.warehouse_sources.sources.featurebase.source import FeaturebaseSource
+from products.warehouse_sources.sources.sdk import SourceInputs
-SOURCE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.featurebase.source"
+SOURCE_MODULE = "products.warehouse_sources.sources.featurebase.source"
def _make_inputs(**overrides: Any) -> SourceInputs:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/webhook_template.py b/products/warehouse_sources/sources/featurebase/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/webhook_template.py
rename to products/warehouse_sources/sources/featurebase/webhook_template.py
index f8ca2307c0e1..7ef5a4bf8c7d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/featurebase/webhook_template.py
+++ b/products/warehouse_sources/sources/featurebase/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
# Featurebase signs webhooks with HMAC-SHA256 over "{timestamp}.{raw body}" using the
# whsec_ signing secret, sent as X-Webhook-Signature with X-Webhook-Timestamp (5 minute
diff --git a/products/warehouse_sources/sources/fintoc/__init__.py b/products/warehouse_sources/sources/fintoc/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/fintoc.py b/products/warehouse_sources/sources/fintoc/_config.py
similarity index 74%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/fintoc.py
rename to products/warehouse_sources/sources/fintoc/_config.py
index 50cb6618699c..fa2d88743ef8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/fintoc.py
+++ b/products/warehouse_sources/sources/fintoc/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/canonical_descriptions.py b/products/warehouse_sources/sources/fintoc/canonical_descriptions.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/canonical_descriptions.py
rename to products/warehouse_sources/sources/fintoc/canonical_descriptions.py
index ded24f1dc1c6..82b14a43f1dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/fintoc/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"payment_intents": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/fintoc.py b/products/warehouse_sources/sources/fintoc/fintoc.py
similarity index 91%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/fintoc.py
rename to products/warehouse_sources/sources/fintoc/fintoc.py
index e620547e51de..401e9f3a77f4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/fintoc.py
+++ b/products/warehouse_sources/sources/fintoc/fintoc.py
@@ -6,38 +6,34 @@ from urllib.parse import parse_qsl, quote, urlencode, urlsplit, urlunsplit
from requests import HTTPError
-from posthog.dataclasses import frozen
-
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- rest_api_resource,
- rest_api_resources,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- ClientConfig,
- EndpointResource,
- RESTAPIConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.fintoc.settings import (
+from products.warehouse_sources.sources.fintoc.settings import (
AUTH_ERROR,
BASE_URL,
ENDPOINTS,
LINK_TOKEN_ERROR,
PERMISSION_ERROR,
)
+from products.warehouse_sources.sources.sdk import (
+ ClientConfig,
+ EndpointResource,
+ ExternalWebhookInfo,
+ Resource,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SourceInputs,
+ SourceResponse,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ frozen,
+ make_tracked_session,
+ rest_api_resource,
+ rest_api_resources,
+)
if TYPE_CHECKING:
import pyarrow as pa
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@frozen
@@ -275,7 +271,7 @@ class FintocAPI:
def webhook_table(table: "pa.Table") -> "pa.Table":
- from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import ( # noqa: PLC0415 - keeps Arrow off the source registry import path
+ from products.warehouse_sources.sources.sdk.internals import ( # noqa: PLC0415 - keeps Arrow off the source registry import path
table_from_py_list,
)
@@ -291,7 +287,7 @@ def webhook_table(table: "pa.Table") -> "pa.Table":
def webhook_template() -> "HogFunctionTemplateDC":
- from products.cdp.backend.templates.hog_function_template import ( # noqa: PLC0415 - keeps template dependencies off the source registry import path
+ from products.warehouse_sources.sources.sdk import ( # noqa: PLC0415 - keeps template dependencies off the source registry import path
HogFunctionTemplateDC,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/settings.py b/products/warehouse_sources/sources/fintoc/settings.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/settings.py
rename to products/warehouse_sources/sources/fintoc/settings.py
index fc3a13f4390f..f6cb8a045e2a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/settings.py
+++ b/products/warehouse_sources/sources/fintoc/settings.py
@@ -1,4 +1,4 @@
-from posthog.dataclasses import frozen
+from products.warehouse_sources.sources.sdk import frozen
BASE_URL = "https://api.fintoc.com"
API_VERSION = "2026-02-01"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/source.py b/products/warehouse_sources/sources/fintoc/source.py
similarity index 82%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/source.py
rename to products/warehouse_sources/sources/fintoc/source.py
index 08b8f7ac12e4..8c9be6a1d569 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/source.py
+++ b/products/warehouse_sources/sources/fintoc/source.py
@@ -2,49 +2,46 @@ from typing import TYPE_CHECKING, cast
from asgiref.sync import async_to_sync
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.fintoc.fintoc import (
+from products.warehouse_sources.sources.fintoc._config import FintocSourceConfig
+from products.warehouse_sources.sources.fintoc.fintoc import (
FintocAPI,
FintocResumeState,
parse_link_tokens,
webhook_table,
webhook_template,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.fintoc.settings import (
+from products.warehouse_sources.sources.fintoc.settings import (
API_VERSION,
AUTH_ERROR,
ENDPOINTS,
LINK_TOKEN_ERROR,
PERMISSION_ERROR,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.fintoc import FintocSourceConfig
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
+ ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSource,
+ WebhookSourceManager,
+ detect,
+)
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@SourceRegistry.register
@@ -125,7 +122,7 @@ class FintocSource(ResumableSource[FintocSourceConfig, FintocResumeState], Webho
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.fintoc.canonical_descriptions import ( # noqa: PLC0415 - loads documentation only when requested
+ from products.warehouse_sources.sources.fintoc.canonical_descriptions import ( # noqa: PLC0415 - loads documentation only when requested
CANONICAL_DESCRIPTIONS,
)
diff --git a/products/warehouse_sources/sources/fintoc/tests/__init__.py b/products/warehouse_sources/sources/fintoc/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/tests/conftest.py b/products/warehouse_sources/sources/fintoc/tests/conftest.py
similarity index 100%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/tests/conftest.py
rename to products/warehouse_sources/sources/fintoc/tests/conftest.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/tests/test_fintoc.py b/products/warehouse_sources/sources/fintoc/tests/test_fintoc.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/tests/test_fintoc.py
rename to products/warehouse_sources/sources/fintoc/tests/test_fintoc.py
index 2c6dcac77bb8..a3d92ff00c3f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/tests/test_fintoc.py
+++ b/products/warehouse_sources/sources/fintoc/tests/test_fintoc.py
@@ -12,18 +12,14 @@ from unittest.mock import MagicMock
import pyarrow as pa
from requests import HTTPError, PreparedRequest
-from products.cdp.backend.validation import compile_hog
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
- RESTClientRetryableError,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.fintoc.fintoc import (
+from products.warehouse_sources.sources.fintoc.fintoc import (
FintocAPI,
FintocResumeState,
webhook_table,
webhook_template,
)
-
-from common.hogvm.python.execute import execute_bytecode
+from products.warehouse_sources.sources.sdk import RESTClientRetryableError
+from products.warehouse_sources.sources.sdk.testing import compile_hog, execute_bytecode
@pytest.mark.parametrize("complete", [False, True])
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/tests/test_fintoc_source.py b/products/warehouse_sources/sources/fintoc/tests/test_fintoc_source.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/tests/test_fintoc_source.py
rename to products/warehouse_sources/sources/fintoc/tests/test_fintoc_source.py
index 5b6bd2f22fba..f78f718fcf04 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fintoc/tests/test_fintoc_source.py
+++ b/products/warehouse_sources/sources/fintoc/tests/test_fintoc_source.py
@@ -5,8 +5,8 @@ from typing import Any, cast
import pytest
from unittest.mock import AsyncMock, MagicMock, patch
-from products.warehouse_sources.backend.temporal.data_imports.sources.fintoc.source import FintocSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.fintoc import FintocSourceConfig
+from products.warehouse_sources.sources.fintoc._config import FintocSourceConfig
+from products.warehouse_sources.sources.fintoc.source import FintocSource
@pytest.mark.parametrize(
diff --git a/products/warehouse_sources/sources/fourthwall/__init__.py b/products/warehouse_sources/sources/fourthwall/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/fourthwall.py b/products/warehouse_sources/sources/fourthwall/_config.py
similarity index 73%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/fourthwall.py
rename to products/warehouse_sources/sources/fourthwall/_config.py
index fabc19be3e66..5b4293b062c2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/fourthwall.py
+++ b/products/warehouse_sources/sources/fourthwall/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/canonical_descriptions.py b/products/warehouse_sources/sources/fourthwall/canonical_descriptions.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/canonical_descriptions.py
rename to products/warehouse_sources/sources/fourthwall/canonical_descriptions.py
index 9592a072f2a1..ae71ba4d55be 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/fourthwall/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
_API_REFERENCE = "https://docs.fourthwall.com/api-reference/platform"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/fourthwall.py b/products/warehouse_sources/sources/fourthwall/fourthwall.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/fourthwall.py
rename to products/warehouse_sources/sources/fourthwall/fourthwall.py
index 49cb0278fe53..1b5e6ce12a5b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/fourthwall.py
+++ b/products/warehouse_sources/sources/fourthwall/fourthwall.py
@@ -8,34 +8,7 @@ from asgiref.sync import async_to_sync
from dateutil import parser
from requests import Request, Response, Session
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import (
- coerce_datetime_to_utc,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- BasePaginator,
- PageNumberPaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- ClientConfig,
- Endpoint,
- EndpointResource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.settings import (
+from products.warehouse_sources.sources.fourthwall.settings import (
ALL_WEBHOOK_EVENTS,
BASE_URL,
FOURTHWALL_ENDPOINTS,
@@ -43,6 +16,25 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall
WEBHOOK_EVENT_DATA_UNWRAP_KEY,
WEBHOOK_SCHEMA_NAMES,
)
+from products.warehouse_sources.sources.sdk import (
+ BasePaginator,
+ ClientConfig,
+ Endpoint,
+ EndpointResource,
+ ExternalWebhookInfo,
+ PageNumberPaginator,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SourceResponse,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSourceManager,
+ WebhookSyncResult,
+ coerce_datetime_to_utc,
+ make_tracked_session,
+ rest_api_resource,
+)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
REQUEST_TIMEOUT_SECONDS = 30
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/settings.py b/products/warehouse_sources/sources/fourthwall/settings.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/settings.py
rename to products/warehouse_sources/sources/fourthwall/settings.py
index 763a48c635a6..eed2b98476bc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/settings.py
+++ b/products/warehouse_sources/sources/fourthwall/settings.py
@@ -1,8 +1,7 @@
from dataclasses import dataclass, field
from typing import Literal
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.sources.sdk import IncrementalField, incremental_field
BASE_URL = "https://api.fourthwall.com"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/source.py b/products/warehouse_sources/sources/fourthwall/source.py
similarity index 83%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/source.py
rename to products/warehouse_sources/sources/fourthwall/source.py
index f2fe8feef0c8..b435308f745d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/source.py
+++ b/products/warehouse_sources/sources/fourthwall/source.py
@@ -1,52 +1,43 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.fourthwall import fourthwall as api_client
+from products.warehouse_sources.sources.fourthwall._config import FourthwallSourceConfig
+from products.warehouse_sources.sources.fourthwall.fourthwall import FourthwallResumeConfig
+from products.warehouse_sources.sources.fourthwall.settings import (
+ ENDPOINTS,
+ INCREMENTAL_ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ SCHEMA_TO_WEBHOOK_EVENTS,
+ SCHEMA_TO_WEBHOOK_RESOURCE,
+ WEBHOOK_SCHEMA_NAMES,
+)
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
build_endpoint_schemas,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall import fourthwall as api_client
-from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.fourthwall import (
- FourthwallResumeConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.settings import (
- ENDPOINTS,
- INCREMENTAL_ENDPOINTS,
- INCREMENTAL_FIELDS,
- SCHEMA_TO_WEBHOOK_EVENTS,
- SCHEMA_TO_WEBHOOK_RESOURCE,
- WEBHOOK_SCHEMA_NAMES,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.fourthwall import (
- FourthwallSourceConfig,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
DEVELOPER_SETTINGS_HELP = (
"In your Fourthwall dashboard go to **Settings > For developers > Open API** and choose "
@@ -73,9 +64,7 @@ class FourthwallSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.webhook_template import ( # noqa: PLC0415
- template,
- )
+ from products.warehouse_sources.sources.fourthwall.webhook_template import template # noqa: PLC0415
return template
@@ -95,7 +84,7 @@ class FourthwallSource(
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.canonical_descriptions import ( # noqa: PLC0415
+ from products.warehouse_sources.sources.fourthwall.canonical_descriptions import ( # noqa: PLC0415
CANONICAL_DESCRIPTIONS,
)
diff --git a/products/warehouse_sources/sources/fourthwall/tests/__init__.py b/products/warehouse_sources/sources/fourthwall/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/tests/test_fourthwall.py b/products/warehouse_sources/sources/fourthwall/tests/test_fourthwall.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/tests/test_fourthwall.py
rename to products/warehouse_sources/sources/fourthwall/tests/test_fourthwall.py
index fbd024516dca..ab873e86b3d4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/tests/test_fourthwall.py
+++ b/products/warehouse_sources/sources/fourthwall/tests/test_fourthwall.py
@@ -7,15 +7,8 @@ from unittest import mock
from requests import Response
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- RecordedRequest,
- ScriptedResponse,
- SourceDriver,
- always,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.fourthwall import (
+from products.warehouse_sources.sources.fourthwall._config import FourthwallSourceConfig
+from products.warehouse_sources.sources.fourthwall.fourthwall import (
FourthwallResumeConfig,
_format_datetime,
create_webhook,
@@ -27,20 +20,23 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall
validate_credentials,
webhook_table_transformer,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.settings import (
+from products.warehouse_sources.sources.fourthwall.settings import (
ALL_WEBHOOK_EVENTS,
ENDPOINTS,
FOURTHWALL_ENDPOINTS,
PAGE_SIZE,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.source import FourthwallSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.fourthwall import (
- FourthwallSourceConfig,
+from products.warehouse_sources.sources.fourthwall.source import FourthwallSource
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import (
+ RecordedRequest,
+ ScriptedResponse,
+ SourceDriver,
+ always,
+ scripted_network,
)
-SESSION_PATCH = (
- "products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.fourthwall.make_tracked_session"
-)
+SESSION_PATCH = "products.warehouse_sources.sources.fourthwall.fourthwall.make_tracked_session"
API_ROOT = "https://api.fourthwall.com/open-api/v1.0"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/tests/test_fourthwall_source.py b/products/warehouse_sources/sources/fourthwall/tests/test_fourthwall_source.py
similarity index 86%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/tests/test_fourthwall_source.py
rename to products/warehouse_sources/sources/fourthwall/tests/test_fourthwall_source.py
index c946ae161b72..f66513e7329e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/tests/test_fourthwall_source.py
+++ b/products/warehouse_sources/sources/fourthwall/tests/test_fourthwall_source.py
@@ -1,18 +1,16 @@
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.settings import (
+from products.warehouse_sources.sources.fourthwall._config import FourthwallSourceConfig
+from products.warehouse_sources.sources.fourthwall.settings import (
SCHEMA_TO_WEBHOOK_EVENTS,
SCHEMA_TO_WEBHOOK_RESOURCE,
WEBHOOK_EVENT_TO_RESOURCE,
WEBHOOK_SCHEMA_NAMES,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.source import FourthwallSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.webhook_template import template
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.fourthwall import (
- FourthwallSourceConfig,
-)
+from products.warehouse_sources.sources.fourthwall.source import FourthwallSource
+from products.warehouse_sources.sources.fourthwall.webhook_template import template
-API_CLIENT_PATCH = "products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.source.api_client"
+API_CLIENT_PATCH = "products.warehouse_sources.sources.fourthwall.source.api_client"
WEBHOOK_URL = "https://us.posthog.com/public/webhooks/abc"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/tests/test_fourthwall_webhook_template.py b/products/warehouse_sources/sources/fourthwall/tests/test_fourthwall_webhook_template.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/tests/test_fourthwall_webhook_template.py
rename to products/warehouse_sources/sources/fourthwall/tests/test_fourthwall_webhook_template.py
index 3ccfb34d3f09..821dbcf2041e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/tests/test_fourthwall_webhook_template.py
+++ b/products/warehouse_sources/sources/fourthwall/tests/test_fourthwall_webhook_template.py
@@ -5,8 +5,8 @@ import hashlib
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateSimpleTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.fourthwall.webhook_template import template
+from products.warehouse_sources.sources.fourthwall.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
SOURCE_ID = "source_test_123"
SIGNING_SECRET = "e3f93c7c-c92b-4b8f-a9b1-5b70e0891abc"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/webhook_template.py b/products/warehouse_sources/sources/fourthwall/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/webhook_template.py
rename to products/warehouse_sources/sources/fourthwall/webhook_template.py
index 34d69fbae6e2..7d4f281bc55c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/fourthwall/webhook_template.py
+++ b/products/warehouse_sources/sources/fourthwall/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/sources/gitea/__init__.py b/products/warehouse_sources/sources/gitea/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/gitea.py b/products/warehouse_sources/sources/gitea/_config.py
similarity index 74%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/gitea.py
rename to products/warehouse_sources/sources/gitea/_config.py
index 54265043bd62..48c179fbf253 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/gitea.py
+++ b/products/warehouse_sources/sources/gitea/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/canonical_descriptions.py b/products/warehouse_sources/sources/gitea/canonical_descriptions.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gitea/canonical_descriptions.py
rename to products/warehouse_sources/sources/gitea/canonical_descriptions.py
index 61e8f825200c..881a66c0a9b6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/gitea/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"issues": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/gitea.py b/products/warehouse_sources/sources/gitea/gitea.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gitea/gitea.py
rename to products/warehouse_sources/sources/gitea/gitea.py
index 0182cf33897f..f1e87c2fc4bf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/gitea.py
+++ b/products/warehouse_sources/sources/gitea/gitea.py
@@ -11,26 +11,19 @@ from asgiref.sync import async_to_sync
from structlog.types import FilteringBoundLogger
from tenacity import retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from posthog.dataclasses import frozen
-
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.gitea.settings import GITEA_ENDPOINTS, PAGE_SIZE, GiteaEndpointConfig
+from products.warehouse_sources.sources.sdk import (
ExternalWebhookInfo,
+ ResumableSourceManager,
+ SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
+ WebhookSourceManager,
WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
+ frozen,
make_tracked_session,
retry_with_deadline,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.gitea.settings import (
- GITEA_ENDPOINTS,
- PAGE_SIZE,
- GiteaEndpointConfig,
-)
GITEA_API_PATH = "/api/v1"
REQUEST_TIMEOUT_SECONDS = 60
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/settings.py b/products/warehouse_sources/sources/gitea/settings.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gitea/settings.py
rename to products/warehouse_sources/sources/gitea/settings.py
index 25ced33e2908..b4c5780cbde5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/settings.py
+++ b/products/warehouse_sources/sources/gitea/settings.py
@@ -1,7 +1,7 @@
from dataclasses import dataclass, field
from typing import Literal, Optional
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType
# Gitea's default MAX_RESPONSE_ITEMS is 50 — instances clamp larger `limit` values to it,
# so asking for more only produces confusing short pages.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/source.py b/products/warehouse_sources/sources/gitea/source.py
similarity index 86%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gitea/source.py
rename to products/warehouse_sources/sources/gitea/source.py
index b829494c89de..2d2532597b8d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/source.py
+++ b/products/warehouse_sources/sources/gitea/source.py
@@ -1,38 +1,13 @@
import secrets
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
+from products.warehouse_sources.sources.sdk import detect
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSource,
- WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import ValidateDatabaseHostMixin
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.gitea import GiteaSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.gitea.gitea import (
+from products.warehouse_sources.sources.gitea._config import GiteaSourceConfig
+from products.warehouse_sources.sources.gitea.gitea import (
GiteaResumeConfig,
create_repo_webhook,
delete_repo_webhook,
@@ -42,12 +17,30 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.gitea.gite
update_repo_webhook_events,
validate_credentials as validate_gitea_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.gitea.settings import (
- ENDPOINTS,
- GITEA_ENDPOINTS,
- INCREMENTAL_FIELDS,
+from products.warehouse_sources.sources.gitea.settings import ENDPOINTS, GITEA_ENDPOINTS, INCREMENTAL_FIELDS
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
+ ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
+ ValidateDatabaseHostMixin,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSource,
+ WebhookSourceManager,
+ WebhookSyncResult,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
# Schemas that can be fed by Gitea webhooks, mapped to the X-Gitea-Event header value the
# webhook handler routes on (schema_mapping is keyed by these values).
@@ -79,7 +72,7 @@ class GiteaSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.gitea.webhook_template import template
+ from products.warehouse_sources.sources.gitea.webhook_template import template
return template
@@ -156,9 +149,7 @@ If automatic creation failed, your token needs admin access to the repository
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.gitea.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.gitea.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/gitea/tests/__init__.py b/products/warehouse_sources/sources/gitea/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/tests/test_gitea.py b/products/warehouse_sources/sources/gitea/tests/test_gitea.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gitea/tests/test_gitea.py
rename to products/warehouse_sources/sources/gitea/tests/test_gitea.py
index ac1aefdb65cc..e0f042a61511 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/tests/test_gitea.py
+++ b/products/warehouse_sources/sources/gitea/tests/test_gitea.py
@@ -7,13 +7,8 @@ from unittest import mock
import pyarrow as pa
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.gitea import GiteaSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.gitea.gitea import (
+from products.warehouse_sources.sources.gitea._config import GiteaSourceConfig
+from products.warehouse_sources.sources.gitea.gitea import (
GiteaResumeConfig,
_make_webhook_dedupe_transformer,
_parse_next_url,
@@ -26,9 +21,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.gitea.gite
update_repo_webhook_events,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.gitea.source import GiteaSource
+from products.warehouse_sources.sources.gitea.source import GiteaSource
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, scripted_network
-_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.gitea.gitea"
+_MODULE = "products.warehouse_sources.sources.gitea.gitea"
BASE_URL = "https://gitea.example.com"
REPO = "owner/repo"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/tests/test_gitea_source.py b/products/warehouse_sources/sources/gitea/tests/test_gitea_source.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gitea/tests/test_gitea_source.py
rename to products/warehouse_sources/sources/gitea/tests/test_gitea_source.py
index c25ed414be23..3cf256bba44e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/tests/test_gitea_source.py
+++ b/products/warehouse_sources/sources/gitea/tests/test_gitea_source.py
@@ -1,13 +1,10 @@
import pytest
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.gitea import GiteaSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.gitea.source import (
- GITEA_WEBHOOK_RESOURCE_MAP,
- GiteaSource,
-)
+from products.warehouse_sources.sources.gitea._config import GiteaSourceConfig
+from products.warehouse_sources.sources.gitea.source import GITEA_WEBHOOK_RESOURCE_MAP, GiteaSource
-_SOURCE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.gitea.source"
+_SOURCE_MODULE = "products.warehouse_sources.sources.gitea.source"
class TestGiteaSource:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/tests/test_gitea_webhook_template.py b/products/warehouse_sources/sources/gitea/tests/test_gitea_webhook_template.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gitea/tests/test_gitea_webhook_template.py
rename to products/warehouse_sources/sources/gitea/tests/test_gitea_webhook_template.py
index 523f5e40a782..3ac612b5c28c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/tests/test_gitea_webhook_template.py
+++ b/products/warehouse_sources/sources/gitea/tests/test_gitea_webhook_template.py
@@ -3,11 +3,11 @@ import json
import hashlib
from typing import Any
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.gitea.webhook_template import template
+from products.warehouse_sources.sources.gitea.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
-class TestGiteaWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestGiteaWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/webhook_template.py b/products/warehouse_sources/sources/gitea/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gitea/webhook_template.py
rename to products/warehouse_sources/sources/gitea/webhook_template.py
index b911ae6b6ad8..d0825ec547e9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gitea/webhook_template.py
+++ b/products/warehouse_sources/sources/gitea/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/sources/gumroad/__init__.py b/products/warehouse_sources/sources/gumroad/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/gumroad.py b/products/warehouse_sources/sources/gumroad/_config.py
similarity index 72%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/gumroad.py
rename to products/warehouse_sources/sources/gumroad/_config.py
index 3dca8f163031..dfe201806a99 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/gumroad.py
+++ b/products/warehouse_sources/sources/gumroad/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/canonical_descriptions.py b/products/warehouse_sources/sources/gumroad/canonical_descriptions.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/canonical_descriptions.py
rename to products/warehouse_sources/sources/gumroad/canonical_descriptions.py
index 6e3f22b2055b..645fa293bd78 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/gumroad/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"sales": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/gumroad.py b/products/warehouse_sources/sources/gumroad/gumroad.py
similarity index 92%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/gumroad.py
rename to products/warehouse_sources/sources/gumroad/gumroad.py
index 141b7187c82b..a8ae3ed80d3a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/gumroad.py
+++ b/products/warehouse_sources/sources/gumroad/gumroad.py
@@ -5,31 +5,7 @@ from typing import Any, Optional, cast
import requests
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.datetime_utils import (
- coerce_datetime_to_utc,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
- build_dependent_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- BasePaginator,
- JSONResponseCursorPaginator,
- SinglePagePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- ClientConfig,
- Endpoint,
- EndpointResource,
- IncrementalConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.settings import (
+from products.warehouse_sources.sources.gumroad.settings import (
EARNINGS_ENDPOINT,
EARNINGS_FIRST_YEAR,
GUMROAD_BASE_URL,
@@ -39,6 +15,22 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.se
PAGE_KEY_PARAM,
GumroadEndpointConfig,
)
+from products.warehouse_sources.sources.sdk import (
+ BasePaginator,
+ ClientConfig,
+ Endpoint,
+ EndpointResource,
+ IncrementalConfig,
+ JSONResponseCursorPaginator,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SinglePagePaginator,
+ SourceResponse,
+ build_dependent_resource,
+ coerce_datetime_to_utc,
+ make_tracked_session,
+ rest_api_resource,
+)
REQUEST_TIMEOUT_SECONDS = 30.0
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/settings.py b/products/warehouse_sources/sources/gumroad/settings.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/settings.py
rename to products/warehouse_sources/sources/gumroad/settings.py
index f83e1faba09c..5af5fa2da300 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/settings.py
+++ b/products/warehouse_sources/sources/gumroad/settings.py
@@ -1,12 +1,6 @@
from dataclasses import field
-from posthog.dataclasses import frozen
-
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
- DependentEndpointConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.sources.sdk import DependentEndpointConfig, IncrementalField, frozen, incremental_field
GUMROAD_BASE_URL = "https://api.gumroad.com"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/source.py b/products/warehouse_sources/sources/gumroad/source.py
similarity index 80%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/source.py
rename to products/warehouse_sources/sources/gumroad/source.py
index e0f5cd7b5c86..e2e9312b6cb9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/source.py
+++ b/products/warehouse_sources/sources/gumroad/source.py
@@ -1,39 +1,31 @@
from typing import Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.gumroad._config import GumroadSourceConfig
+from products.warehouse_sources.sources.gumroad.gumroad import (
+ GumroadResumeConfig,
+ check_endpoint_permission,
+ gumroad_source,
+ validate_credentials as validate_gumroad_credentials,
+)
+from products.warehouse_sources.sources.gumroad.settings import ENDPOINTS, GUMROAD_ENDPOINTS, INCREMENTAL_FIELDS
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import FieldType, ResumableSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
SourceSchema,
build_endpoint_schemas,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.gumroad import (
- GumroadSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.gumroad import (
- GumroadResumeConfig,
- check_endpoint_permission,
- gumroad_source,
- validate_credentials as validate_gumroad_credentials,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.settings import (
- ENDPOINTS,
- GUMROAD_ENDPOINTS,
- INCREMENTAL_FIELDS,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
@SourceRegistry.register
@@ -62,9 +54,7 @@ class GumroadSource(ResumableSource[GumroadSourceConfig, GumroadResumeConfig]):
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.gumroad.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/gumroad/tests/__init__.py b/products/warehouse_sources/sources/gumroad/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/tests/test_gumroad.py b/products/warehouse_sources/sources/gumroad/tests/test_gumroad.py
similarity index 91%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/tests/test_gumroad.py
rename to products/warehouse_sources/sources/gumroad/tests/test_gumroad.py
index 7604355ec402..6da280587d6f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/tests/test_gumroad.py
+++ b/products/warehouse_sources/sources/gumroad/tests/test_gumroad.py
@@ -8,20 +8,8 @@ from unittest.mock import patch
import requests
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- JSONResponseCursorPaginator,
- SinglePagePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
- always,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.gumroad import (
- GumroadSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.gumroad import (
+from products.warehouse_sources.sources.gumroad._config import GumroadSourceConfig
+from products.warehouse_sources.sources.gumroad.gumroad import (
GumroadResumeConfig,
_format_gumroad_date,
_paginator_for,
@@ -31,8 +19,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.gu
gumroad_source,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.settings import GUMROAD_ENDPOINTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.source import GumroadSource
+from products.warehouse_sources.sources.gumroad.settings import GUMROAD_ENDPOINTS
+from products.warehouse_sources.sources.gumroad.source import GumroadSource
+from products.warehouse_sources.sources.sdk import JSONResponseCursorPaginator, SinglePagePaginator
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network
class TestGumroadTransport:
@@ -95,7 +85,7 @@ class TestGumroadTransport:
("payouts", ["id"], "created_at"),
]
)
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.gumroad.rest_api_resource")
+ @patch("products.warehouse_sources.sources.gumroad.gumroad.rest_api_resource")
def test_top_level_source_response_shape(self, endpoint, primary_keys, partition_key, _mock) -> None:
response = gumroad_source(access_token="tok", endpoint=endpoint, team_id=1, job_id="job-1")
assert response.primary_keys == primary_keys
@@ -112,7 +102,7 @@ class TestGumroadTransport:
("subscribers", ["id"]),
]
)
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.gumroad.build_dependent_resource")
+ @patch("products.warehouse_sources.sources.gumroad.gumroad.build_dependent_resource")
def test_fanout_primary_keys_are_unique_table_wide(self, endpoint, primary_keys, mock_build) -> None:
# Universal offer codes and global custom fields are repeated under every product they
# apply to, so the parent product id has to be part of the key.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/tests/test_gumroad_source.py b/products/warehouse_sources/sources/gumroad/tests/test_gumroad_source.py
similarity index 82%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/tests/test_gumroad_source.py
rename to products/warehouse_sources/sources/gumroad/tests/test_gumroad_source.py
index e5aec8723a1b..7078a547c81d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/gumroad/tests/test_gumroad_source.py
+++ b/products/warehouse_sources/sources/gumroad/tests/test_gumroad_source.py
@@ -1,11 +1,9 @@
import pytest
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.gumroad import (
- GumroadSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.settings import GUMROAD_ENDPOINTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.source import GumroadSource
+from products.warehouse_sources.sources.gumroad._config import GumroadSourceConfig
+from products.warehouse_sources.sources.gumroad.settings import GUMROAD_ENDPOINTS
+from products.warehouse_sources.sources.gumroad.source import GumroadSource
class TestGumroadSource:
@@ -37,9 +35,7 @@ class TestGumroadSource:
def test_primary_keys(self, name: str, expected_keys: list[str]) -> None:
assert GUMROAD_ENDPOINTS[name].primary_key == expected_keys
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.gumroad.source.check_endpoint_permission"
- )
+ @mock.patch("products.warehouse_sources.sources.gumroad.source.check_endpoint_permission")
def test_get_endpoint_permissions_reports_missing_scope_per_table(self, mock_probe: mock.MagicMock) -> None:
# A token can hold view_public without view_payouts, and the user needs to see which
# table that costs them rather than having source creation fail.
diff --git a/products/warehouse_sources/sources/instantly/__init__.py b/products/warehouse_sources/sources/instantly/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/instantly.py b/products/warehouse_sources/sources/instantly/_config.py
similarity index 71%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/instantly.py
rename to products/warehouse_sources/sources/instantly/_config.py
index 8dea46219a37..5464a759c8c7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/instantly.py
+++ b/products/warehouse_sources/sources/instantly/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/canonical_descriptions.py b/products/warehouse_sources/sources/instantly/canonical_descriptions.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/instantly/canonical_descriptions.py
rename to products/warehouse_sources/sources/instantly/canonical_descriptions.py
index 1d8c8454d958..61a398028000 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/instantly/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"campaigns": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/instantly.py b/products/warehouse_sources/sources/instantly/instantly.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/instantly/instantly.py
rename to products/warehouse_sources/sources/instantly/instantly.py
index 677d20fd3364..631681491114 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/instantly.py
+++ b/products/warehouse_sources/sources/instantly/instantly.py
@@ -13,34 +13,7 @@ import requests
from asgiref.sync import async_to_sync
from structlog.types import FilteringBoundLogger
-from posthog.dataclasses import frozen
-
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- RESTClient,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import BearerTokenAuth
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- BasePaginator,
- JSONResponseCursorPaginator,
- SinglePagePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- Endpoint,
- EndpointResource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.instantly.settings import (
+from products.warehouse_sources.sources.instantly.settings import (
ANALYTICS_HISTORY_START_DATE,
BASE_URL,
DEFAULT_PAGE_SIZE,
@@ -50,6 +23,26 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instantly.
WEBHOOK_EVENTS_ENDPOINT,
InstantlyEndpointConfig,
)
+from products.warehouse_sources.sources.sdk import (
+ BasePaginator,
+ BearerTokenAuth,
+ Endpoint,
+ EndpointResource,
+ ExternalWebhookInfo,
+ JSONResponseCursorPaginator,
+ RESTAPIConfig,
+ RESTClient,
+ ResumableSourceManager,
+ SinglePagePaginator,
+ SourceResponse,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSourceManager,
+ frozen,
+ make_tracked_session,
+ rest_api_resource,
+)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
REQUEST_TIMEOUT_SECONDS = 30
# (connect, read). Analytics responses can span years of rows, so reads get more room than probes.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/settings.py b/products/warehouse_sources/sources/instantly/settings.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/instantly/settings.py
rename to products/warehouse_sources/sources/instantly/settings.py
index 84b55374a7ee..b7ccbd1b5b98 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/settings.py
+++ b/products/warehouse_sources/sources/instantly/settings.py
@@ -1,8 +1,7 @@
from dataclasses import dataclass, field
from typing import Any, Literal, Optional
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType, incremental_field
BASE_URL = "https://api.instantly.ai"
API_VERSION = "v2"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/source.py b/products/warehouse_sources/sources/instantly/source.py
similarity index 84%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/instantly/source.py
rename to products/warehouse_sources/sources/instantly/source.py
index 37133c2eae3a..c1e0e8b5916f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/source.py
+++ b/products/warehouse_sources/sources/instantly/source.py
@@ -2,41 +2,13 @@ from typing import TYPE_CHECKING, Optional, cast
import structlog
-from products.warehouse_sources.backend.facade.codebase_detection import detect
+from products.warehouse_sources.sources.sdk import detect
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
- build_endpoint_schemas,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.instantly import (
- InstantlySourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.instantly.instantly import (
+from products.warehouse_sources.sources.instantly._config import InstantlySourceConfig
+from products.warehouse_sources.sources.instantly.instantly import (
InstantlyResumeConfig,
create_webhook as create_instantly_webhook,
delete_webhook as delete_instantly_webhook,
@@ -45,14 +17,35 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instantly.
instantly_source,
validate_credentials as validate_instantly_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.instantly.settings import (
+from products.warehouse_sources.sources.instantly.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
INSTANTLY_ENDPOINTS,
WEBHOOK_EVENTS_ENDPOINT,
WEBHOOK_ROUTING_KEY,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
+ ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSource,
+ WebhookSourceManager,
+ build_endpoint_schemas,
+)
# Webhook management calls run in the API request path where no job logger exists.
logger = structlog.get_logger(__name__)
@@ -134,7 +127,7 @@ Instantly webhooks require the Hypergrowth plan or above, and automatic creation
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.instantly.canonical_descriptions import ( # noqa: PLC0415
+ from products.warehouse_sources.sources.instantly.canonical_descriptions import ( # noqa: PLC0415
CANONICAL_DESCRIPTIONS,
)
@@ -142,9 +135,7 @@ Instantly webhooks require the Hypergrowth plan or above, and automatic creation
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.instantly.webhook_template import ( # noqa: PLC0415
- template,
- )
+ from products.warehouse_sources.sources.instantly.webhook_template import template # noqa: PLC0415
return template
diff --git a/products/warehouse_sources/sources/instantly/tests/__init__.py b/products/warehouse_sources/sources/instantly/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly.py b/products/warehouse_sources/sources/instantly/tests/test_instantly.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly.py
rename to products/warehouse_sources/sources/instantly/tests/test_instantly.py
index c3185ada1dd9..4baa13d1b1a4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly.py
+++ b/products/warehouse_sources/sources/instantly/tests/test_instantly.py
@@ -7,8 +7,7 @@ from unittest import mock
from requests import Request
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.instantly.instantly import (
+from products.warehouse_sources.sources.instantly.instantly import (
SYNC_REQUEST_TIMEOUT_SECONDS,
WEBHOOK_PLAN_ERROR,
WEBHOOK_SECRET_HEADER,
@@ -26,12 +25,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.instantly.
instantly_source,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.instantly.settings import (
- ANALYTICS_HISTORY_START_DATE,
- PROBE_PARENT_ID,
-)
+from products.warehouse_sources.sources.instantly.settings import ANALYTICS_HISTORY_START_DATE, PROBE_PARENT_ID
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
-MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.instantly.instantly"
+MODULE = "products.warehouse_sources.sources.instantly.instantly"
class _FakeClient:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly_source.py b/products/warehouse_sources/sources/instantly/tests/test_instantly_source.py
similarity index 84%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly_source.py
rename to products/warehouse_sources/sources/instantly/tests/test_instantly_source.py
index 94f4e4fd8a22..5f4e8698b53a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/tests/test_instantly_source.py
+++ b/products/warehouse_sources/sources/instantly/tests/test_instantly_source.py
@@ -1,13 +1,8 @@
import pytest
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.instantly import (
- InstantlySourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.instantly.settings import (
- ENDPOINTS,
- WEBHOOK_EVENTS_ENDPOINT,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.instantly.source import InstantlySource
+from products.warehouse_sources.sources.instantly._config import InstantlySourceConfig
+from products.warehouse_sources.sources.instantly.settings import ENDPOINTS, WEBHOOK_EVENTS_ENDPOINT
+from products.warehouse_sources.sources.instantly.source import InstantlySource
class TestInstantlySource:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/webhook_template.py b/products/warehouse_sources/sources/instantly/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/instantly/webhook_template.py
rename to products/warehouse_sources/sources/instantly/webhook_template.py
index 6542fcfa4c21..05553683f9d1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/instantly/webhook_template.py
+++ b/products/warehouse_sources/sources/instantly/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
# Instantly does not sign webhook deliveries, but it does send static custom headers configured
# on the webhook. `create_webhook` attaches a generated secret in the x-posthog-webhook-secret
diff --git a/products/warehouse_sources/sources/lemon_squeezy/__init__.py b/products/warehouse_sources/sources/lemon_squeezy/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/lemonsqueezy.py b/products/warehouse_sources/sources/lemon_squeezy/_config.py
similarity index 72%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/lemonsqueezy.py
rename to products/warehouse_sources/sources/lemon_squeezy/_config.py
index 7f94b553b95d..8f4c48086ba6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/lemonsqueezy.py
+++ b/products/warehouse_sources/sources/lemon_squeezy/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/canonical_descriptions.py b/products/warehouse_sources/sources/lemon_squeezy/canonical_descriptions.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/canonical_descriptions.py
rename to products/warehouse_sources/sources/lemon_squeezy/canonical_descriptions.py
index f4fd2fa3d82d..5c70d9f93237 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/lemon_squeezy/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"stores": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/lemon_squeezy.py b/products/warehouse_sources/sources/lemon_squeezy/lemon_squeezy.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/lemon_squeezy.py
rename to products/warehouse_sources/sources/lemon_squeezy/lemon_squeezy.py
index 69e24092d236..67ae17dd6a56 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/lemon_squeezy.py
+++ b/products/warehouse_sources/sources/lemon_squeezy/lemon_squeezy.py
@@ -10,26 +10,7 @@ import pyarrow as pa
from asgiref.sync import async_to_sync
from requests import Response, Session
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- JSONResponsePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_squeezy.settings import (
+from products.warehouse_sources.sources.lemon_squeezy.settings import (
ALL_WEBHOOK_EVENTS,
BASE_URL,
JSON_API_HEADERS,
@@ -37,6 +18,21 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_sque
PAGE_SIZE,
WEBHOOK_SCHEMA_NAMES,
)
+from products.warehouse_sources.sources.sdk import (
+ ExternalWebhookInfo,
+ JSONResponsePaginator,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SourceResponse,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSourceManager,
+ WebhookSyncResult,
+ make_tracked_session,
+ rest_api_resource,
+ validate_via_probe,
+)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
@dataclasses.dataclass
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/settings.py b/products/warehouse_sources/sources/lemon_squeezy/settings.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/settings.py
rename to products/warehouse_sources/sources/lemon_squeezy/settings.py
index 4e7ad24e82aa..6d05db41d290 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/settings.py
+++ b/products/warehouse_sources/sources/lemon_squeezy/settings.py
@@ -1,7 +1,6 @@
from dataclasses import dataclass, field
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.sources.sdk import IncrementalField, incremental_field
BASE_URL = "https://api.lemonsqueezy.com"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/source.py b/products/warehouse_sources/sources/lemon_squeezy/source.py
similarity index 83%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/source.py
rename to products/warehouse_sources/sources/lemon_squeezy/source.py
index 943cae8f8808..da2569fa9848 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/source.py
+++ b/products/warehouse_sources/sources/lemon_squeezy/source.py
@@ -1,52 +1,43 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.lemon_squeezy import lemon_squeezy as api_client
+from products.warehouse_sources.sources.lemon_squeezy._config import LemonSqueezySourceConfig
+from products.warehouse_sources.sources.lemon_squeezy.lemon_squeezy import LemonSqueezyResumeConfig
+from products.warehouse_sources.sources.lemon_squeezy.settings import (
+ ENDPOINTS,
+ INCREMENTAL_ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ RESOURCE_TO_JSON_API_TYPE,
+ SCHEMA_TO_WEBHOOK_EVENTS,
+ WEBHOOK_SCHEMA_NAMES,
+)
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
build_endpoint_schemas,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.lemonsqueezy import (
- LemonSqueezySourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_squeezy import lemon_squeezy as api_client
-from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_squeezy.lemon_squeezy import (
- LemonSqueezyResumeConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_squeezy.settings import (
- ENDPOINTS,
- INCREMENTAL_ENDPOINTS,
- INCREMENTAL_FIELDS,
- RESOURCE_TO_JSON_API_TYPE,
- SCHEMA_TO_WEBHOOK_EVENTS,
- WEBHOOK_SCHEMA_NAMES,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
LEMON_SQUEEZY_API_SETTINGS_URL = "https://app.lemonsqueezy.com/settings/api"
@@ -72,9 +63,7 @@ class LemonSqueezySource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_squeezy.webhook_template import (
- template,
- )
+ from products.warehouse_sources.sources.lemon_squeezy.webhook_template import template
return template
@@ -162,9 +151,7 @@ class LemonSqueezySource(
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_squeezy.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.lemon_squeezy.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/lemon_squeezy/tests/__init__.py b/products/warehouse_sources/sources/lemon_squeezy/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/tests/test_lemon_squeezy.py b/products/warehouse_sources/sources/lemon_squeezy/tests/test_lemon_squeezy.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/tests/test_lemon_squeezy.py
rename to products/warehouse_sources/sources/lemon_squeezy/tests/test_lemon_squeezy.py
index 50847166a952..c1a592e392da 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/tests/test_lemon_squeezy.py
+++ b/products/warehouse_sources/sources/lemon_squeezy/tests/test_lemon_squeezy.py
@@ -3,16 +3,8 @@ from typing import Any
import pytest
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.lemonsqueezy import (
- LemonSqueezySourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_squeezy.lemon_squeezy import (
+from products.warehouse_sources.sources.lemon_squeezy._config import LemonSqueezySourceConfig
+from products.warehouse_sources.sources.lemon_squeezy.lemon_squeezy import (
LemonSqueezyResumeConfig,
LemonSqueezyUntrustedURLError,
_assert_lemon_squeezy_origin,
@@ -25,11 +17,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_sque
sync_webhook_events,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_squeezy.settings import (
- ALL_WEBHOOK_EVENTS,
- BASE_URL,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_squeezy.source import LemonSqueezySource
+from products.warehouse_sources.sources.lemon_squeezy.settings import ALL_WEBHOOK_EVENTS, BASE_URL
+from products.warehouse_sources.sources.lemon_squeezy.source import LemonSqueezySource
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, scripted_network
def _json_api_item(resource_id: str, created_at: str, **attributes: Any) -> dict[str, Any]:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/tests/test_lemon_squeezy_source.py b/products/warehouse_sources/sources/lemon_squeezy/tests/test_lemon_squeezy_source.py
similarity index 88%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/tests/test_lemon_squeezy_source.py
rename to products/warehouse_sources/sources/lemon_squeezy/tests/test_lemon_squeezy_source.py
index 19ad834a3884..666c16dada69 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/tests/test_lemon_squeezy_source.py
+++ b/products/warehouse_sources/sources/lemon_squeezy/tests/test_lemon_squeezy_source.py
@@ -1,15 +1,11 @@
import pytest
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.lemonsqueezy import (
- LemonSqueezySourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_squeezy.settings import (
- SCHEMA_TO_WEBHOOK_EVENTS,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.lemon_squeezy.source import LemonSqueezySource
-
-API_CLIENT_PATCH = "products.warehouse_sources.backend.temporal.data_imports.sources.lemon_squeezy.source.api_client"
+from products.warehouse_sources.sources.lemon_squeezy._config import LemonSqueezySourceConfig
+from products.warehouse_sources.sources.lemon_squeezy.settings import SCHEMA_TO_WEBHOOK_EVENTS
+from products.warehouse_sources.sources.lemon_squeezy.source import LemonSqueezySource
+
+API_CLIENT_PATCH = "products.warehouse_sources.sources.lemon_squeezy.source.api_client"
class TestLemonSqueezySource:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/webhook_template.py b/products/warehouse_sources/sources/lemon_squeezy/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/webhook_template.py
rename to products/warehouse_sources/sources/lemon_squeezy/webhook_template.py
index 600e2351f42d..09c9f5ae6fb5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/lemon_squeezy/webhook_template.py
+++ b/products/warehouse_sources/sources/lemon_squeezy/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/sources/mailerlite/__init__.py b/products/warehouse_sources/sources/mailerlite/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mailerlite.py b/products/warehouse_sources/sources/mailerlite/_config.py
similarity index 72%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mailerlite.py
rename to products/warehouse_sources/sources/mailerlite/_config.py
index bdab90dba0f5..88e64d749ee1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mailerlite.py
+++ b/products/warehouse_sources/sources/mailerlite/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/api_inventory.md b/products/warehouse_sources/sources/mailerlite/api_inventory.md
similarity index 100%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/api_inventory.md
rename to products/warehouse_sources/sources/mailerlite/api_inventory.md
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/canonical_descriptions.py b/products/warehouse_sources/sources/mailerlite/canonical_descriptions.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/canonical_descriptions.py
rename to products/warehouse_sources/sources/mailerlite/canonical_descriptions.py
index 03c70cf4dd43..4784451de390 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/mailerlite/canonical_descriptions.py
@@ -5,9 +5,7 @@ Keyed by the endpoint names in `settings.py` `MAILERLITE_ENDPOINTS`, which match
`ExternalDataSchema.name` of a synced MailerLite table. Columns absent here fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"subscribers": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/mailerlite.py b/products/warehouse_sources/sources/mailerlite/mailerlite.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/mailerlite.py
rename to products/warehouse_sources/sources/mailerlite/mailerlite.py
index dab9c1fea0ac..b79555fd3c93 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/mailerlite.py
+++ b/products/warehouse_sources/sources/mailerlite/mailerlite.py
@@ -9,30 +9,7 @@ import pyarrow as pa
from asgiref.sync import async_to_sync
from requests import Response, Session
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
- build_dependent_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- JSONResponsePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.settings import (
+from products.warehouse_sources.sources.mailerlite.settings import (
ALL_WEBHOOK_EVENTS,
API_VERSION_HEADERS,
MAILERLITE_ENDPOINTS,
@@ -41,6 +18,23 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite
WEBHOOK_SCHEMA_NAMES,
MailerLiteEndpointConfig,
)
+from products.warehouse_sources.sources.sdk import (
+ ClientConfig,
+ ExternalWebhookInfo,
+ JSONResponsePaginator,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SourceResponse,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSourceManager,
+ WebhookSyncResult,
+ build_dependent_resource,
+ make_tracked_session,
+ rest_api_resource,
+ validate_via_probe,
+)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
MAILERLITE_BASE_URL = "https://connect.mailerlite.com/api"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/settings.py b/products/warehouse_sources/sources/mailerlite/settings.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/settings.py
rename to products/warehouse_sources/sources/mailerlite/settings.py
index d3cd6506eeae..f649d5c9638b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/settings.py
+++ b/products/warehouse_sources/sources/mailerlite/settings.py
@@ -2,10 +2,7 @@ from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any, Optional
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
- DependentEndpointConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ResponseAction
+from products.warehouse_sources.sources.sdk import DependentEndpointConfig, ResponseAction
# MailerLite caps list endpoints at 100 rows per page; default is 25.
PAGE_SIZE = 100
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/source.py b/products/warehouse_sources/sources/mailerlite/source.py
similarity index 84%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/source.py
rename to products/warehouse_sources/sources/mailerlite/source.py
index 09b616566674..ae25a8ccaff0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/source.py
+++ b/products/warehouse_sources/sources/mailerlite/source.py
@@ -1,44 +1,13 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
- VersionDeprecation,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSource,
- WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
- build_endpoint_schemas,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mailerlite import (
- MailerLiteSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite import mailerlite as api_client
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.mailerlite import (
+from products.warehouse_sources.sources.mailerlite import mailerlite as api_client
+from products.warehouse_sources.sources.mailerlite._config import MailerLiteSourceConfig
+from products.warehouse_sources.sources.mailerlite.mailerlite import (
MailerLiteResumeConfig,
mailerlite_source,
validate_credentials as validate_mailerlite_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.settings import (
+from products.warehouse_sources.sources.mailerlite.settings import (
DEFAULT_VERSION,
ENDPOINTS,
MAILERLITE_ENDPOINTS,
@@ -48,10 +17,34 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite
WEBHOOK_RESOURCE_MAP,
WEBHOOK_SCHEMA_NAMES,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
+ ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
+ VersionDeprecation,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSource,
+ WebhookSourceManager,
+ WebhookSyncResult,
+ build_endpoint_schemas,
+ detect,
+)
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@SourceRegistry.register
@@ -90,9 +83,7 @@ class MailerLiteSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.webhook_template import (
- template,
- )
+ from products.warehouse_sources.sources.mailerlite.webhook_template import template
return template
@@ -165,9 +156,7 @@ Copy the `secret` from the response into the field below. MailerLite never retur
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.mailerlite.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/mailerlite/tests/__init__.py b/products/warehouse_sources/sources/mailerlite/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/tests/test_mailerlite.py b/products/warehouse_sources/sources/mailerlite/tests/test_mailerlite.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/tests/test_mailerlite.py
rename to products/warehouse_sources/sources/mailerlite/tests/test_mailerlite.py
index c3bffc5c017e..98fdb96fbd44 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/tests/test_mailerlite.py
+++ b/products/warehouse_sources/sources/mailerlite/tests/test_mailerlite.py
@@ -5,22 +5,8 @@ from unittest.mock import MagicMock
from requests.exceptions import ChunkedEncodingError, HTTPError
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import (
- RESTClientRetryableError,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
- always,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mailerlite import (
- MailerLiteSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.mailerlite import (
+from products.warehouse_sources.sources.mailerlite._config import MailerLiteSourceConfig
+from products.warehouse_sources.sources.mailerlite.mailerlite import (
INVALID_KEY_ERROR,
MAILERLITE_BASE_URL,
UNVERIFIED_KEY_ERROR,
@@ -33,8 +19,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite
sync_webhook_events,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.settings import MAILERLITE_V2
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.source import MailerLiteSource
+from products.warehouse_sources.sources.mailerlite.settings import MAILERLITE_V2
+from products.warehouse_sources.sources.mailerlite.source import MailerLiteSource
+from products.warehouse_sources.sources.sdk import (
+ RESTClientRetryableError,
+ ResumableSourceManager,
+ WebhookSourceManager,
+)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network
def _make_manager(resume: MailerLiteResumeConfig | None = None) -> MagicMock:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/tests/test_mailerlite_source.py b/products/warehouse_sources/sources/mailerlite/tests/test_mailerlite_source.py
similarity index 78%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/tests/test_mailerlite_source.py
rename to products/warehouse_sources/sources/mailerlite/tests/test_mailerlite_source.py
index 1489fd544fee..8322708f13dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/tests/test_mailerlite_source.py
+++ b/products/warehouse_sources/sources/mailerlite/tests/test_mailerlite_source.py
@@ -3,25 +3,23 @@ from typing import Any
import pytest
from unittest.mock import MagicMock, patch
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mailerlite import (
- MailerLiteSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.settings import (
+from products.warehouse_sources.sources.mailerlite._config import MailerLiteSourceConfig
+from products.warehouse_sources.sources.mailerlite.settings import (
MAILERLITE_V1,
MAILERLITE_V2,
SUBSCRIBER_WEBHOOK_EVENTS,
WEBHOOK_SCHEMA_NAMES,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.source import MailerLiteSource
+from products.warehouse_sources.sources.mailerlite.source import MailerLiteSource
+from products.warehouse_sources.sources.sdk import (
+ ExternalWebhookInfo,
+ ResumableSourceManager,
+ SourceInputs,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSourceManager,
+ WebhookSyncResult,
+)
def _config() -> MailerLiteSourceConfig:
@@ -35,7 +33,7 @@ class TestMailerLiteSourceClass:
)
def test_validate_credentials_passes_the_probe_result_through(self, probe_result: tuple[bool, str | None]) -> None:
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.source.validate_mailerlite_credentials",
+ "products.warehouse_sources.sources.mailerlite.source.validate_mailerlite_credentials",
return_value=probe_result,
):
assert MailerLiteSource().validate_credentials(_config(), team_id=1) == probe_result
@@ -50,7 +48,7 @@ class TestMailerLiteSourceClass:
)
def test_validate_credentials_probes_a_listable_path(self, schema_name: str, expected_path: str) -> None:
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.source.validate_mailerlite_credentials",
+ "products.warehouse_sources.sources.mailerlite.source.validate_mailerlite_credentials",
return_value=(True, None),
) as probe:
MailerLiteSource().validate_credentials(_config(), team_id=1, schema_name=schema_name)
@@ -76,9 +74,7 @@ class TestMailerLiteSourceClass:
inputs.job_id = "job-1"
manager = MagicMock(spec=ResumableSourceManager)
- with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.source.mailerlite_source"
- ) as mock_source:
+ with patch("products.warehouse_sources.sources.mailerlite.source.mailerlite_source") as mock_source:
MailerLiteSource().source_for_pipeline(_config(), manager, inputs)
kwargs = mock_source.call_args.kwargs
assert kwargs["api_key"] == "test-key"
@@ -122,7 +118,7 @@ class TestMailerLiteWebhooks:
# Unwired, these fall through to the base class, which raises or reports "not supported"
# and leaves the user with no webhook at all.
with patch(
- f"products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.mailerlite.{client_function}",
+ f"products.warehouse_sources.sources.mailerlite.mailerlite.{client_function}",
return_value=expected,
) as mock_client:
result = getattr(MailerLiteSource(), method)(_config(), "https://ph.example/webhook", 1)
@@ -132,7 +128,7 @@ class TestMailerLiteWebhooks:
def test_sync_webhook_events_sends_the_eligible_events(self) -> None:
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.mailerlite.mailerlite.sync_webhook_events",
+ "products.warehouse_sources.sources.mailerlite.mailerlite.sync_webhook_events",
return_value=WebhookSyncResult(success=True),
) as mock_sync:
MailerLiteSource().sync_webhook_events(_config(), "https://ph.example/webhook", 1, ["subscribers"])
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/webhook_template.py b/products/warehouse_sources/sources/mailerlite/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/webhook_template.py
rename to products/warehouse_sources/sources/mailerlite/webhook_template.py
index 886a64909b1d..584fb79af7d4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailerlite/webhook_template.py
+++ b/products/warehouse_sources/sources/mailerlite/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
# MailerLite signs every delivery with a `Signature` header holding the lowercase hex
# HMAC-SHA256 of the raw request body, keyed by the `secret` MailerLite generates when the
diff --git a/products/warehouse_sources/sources/mailgun/__init__.py b/products/warehouse_sources/sources/mailgun/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mailgun.py b/products/warehouse_sources/sources/mailgun/_config.py
similarity index 77%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mailgun.py
rename to products/warehouse_sources/sources/mailgun/_config.py
index 46f1f706039b..d0de99c5a931 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mailgun.py
+++ b/products/warehouse_sources/sources/mailgun/_config.py
@@ -3,7 +3,7 @@
from typing import Literal
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/canonical_descriptions.py b/products/warehouse_sources/sources/mailgun/canonical_descriptions.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/canonical_descriptions.py
rename to products/warehouse_sources/sources/mailgun/canonical_descriptions.py
index 98557e87ed90..06b876546bc9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/mailgun/canonical_descriptions.py
@@ -5,9 +5,7 @@ Keyed by the endpoint names in `settings.py` `MAILGUN_ENDPOINTS`, which match th
`ExternalDataSchema.name` of a synced Mailgun table. Columns absent here fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
_METRICS_DOCS_URL = (
"https://documentation.mailgun.com/docs/mailgun/api-reference/send/mailgun/metrics/post-v1-analytics-metrics"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py b/products/warehouse_sources/sources/mailgun/mailgun.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py
rename to products/warehouse_sources/sources/mailgun/mailgun.py
index 0ca757a6128f..db1b8060f0ec 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/mailgun.py
+++ b/products/warehouse_sources/sources/mailgun/mailgun.py
@@ -16,21 +16,7 @@ from asgiref.sync import async_to_sync
from structlog.types import FilteringBoundLogger
from tenacity import RetryCallState, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
- make_tracked_session,
- retry_with_deadline,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.settings import (
+from products.warehouse_sources.sources.mailgun.settings import (
LOGS_ENDPOINT,
LOGS_PATH,
MAILGUN_ENDPOINTS,
@@ -42,6 +28,18 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.se
MailgunEndpointConfig,
MailgunMetricsEndpointConfig,
)
+from products.warehouse_sources.sources.sdk import (
+ ExternalWebhookInfo,
+ ResumableSourceManager,
+ SourceResponse,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSourceManager,
+ WebhookSyncResult,
+ make_tracked_session,
+ retry_with_deadline,
+)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
REGION_BASE_URLS = {
"us": "https://api.mailgun.net",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/settings.py b/products/warehouse_sources/sources/mailgun/settings.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/settings.py
rename to products/warehouse_sources/sources/mailgun/settings.py
index 14aaf12cae03..cbd5b29ff0a9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/settings.py
+++ b/products/warehouse_sources/sources/mailgun/settings.py
@@ -1,8 +1,7 @@
from dataclasses import dataclass, field
from typing import Literal, Optional
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import PartitionFormat
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType, PartitionFormat
PaginationStyle = Literal["paging", "skip"]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/source.py b/products/warehouse_sources/sources/mailgun/source.py
similarity index 86%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/source.py
rename to products/warehouse_sources/sources/mailgun/source.py
index 2ce3f000673d..f57eae23b78f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/source.py
+++ b/products/warehouse_sources/sources/mailgun/source.py
@@ -1,36 +1,7 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
- SourceFieldSelectConfig,
- SourceFieldSelectConfigOption,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSource,
- WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mailgun import (
- MailgunSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.mailgun import (
+from products.warehouse_sources.sources.mailgun._config import MailgunSourceConfig
+from products.warehouse_sources.sources.mailgun.mailgun import (
MailgunResumeConfig,
create_webhook as create_mailgun_webhook,
delete_webhook as delete_mailgun_webhook,
@@ -39,17 +10,41 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.ma
sync_webhook_events as sync_mailgun_webhook_events,
validate_credentials as validate_mailgun_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.settings import (
+from products.warehouse_sources.sources.mailgun.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
SCHEMA_TO_WEBHOOK_RESOURCE,
WEBHOOK_EVENTS_ENDPOINT,
WEBHOOK_TYPES,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
+ ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceFieldSelectConfig,
+ SourceFieldSelectConfigOption,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSource,
+ WebhookSourceManager,
+ WebhookSyncResult,
+ detect,
+)
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
API_SECURITY_URL = "https://app.mailgun.com/settings/api_security"
@@ -87,9 +82,7 @@ class MailgunSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.webhook_template import ( # noqa: PLC0415
- template,
- )
+ from products.warehouse_sources.sources.mailgun.webhook_template import template # noqa: PLC0415
return template
@@ -180,9 +173,7 @@ Note: Mailgun only retains events for a limited period (1 day on free plans, up
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.mailgun.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/mailgun/tests/__init__.py b/products/warehouse_sources/sources/mailgun/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun.py b/products/warehouse_sources/sources/mailgun/tests/test_mailgun.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun.py
rename to products/warehouse_sources/sources/mailgun/tests/test_mailgun.py
index 89b4313bad9f..5e3318291ac0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun.py
+++ b/products/warehouse_sources/sources/mailgun/tests/test_mailgun.py
@@ -10,20 +10,9 @@ from unittest import mock
import requests
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import WebhookCreationResult
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- RecordedRequest,
- ScriptedResponse,
- SourceDriver,
- always,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mailgun import (
- MailgunSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun import mailgun as mailgun_module
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.mailgun import (
+from products.warehouse_sources.sources.mailgun import mailgun as mailgun_module
+from products.warehouse_sources.sources.mailgun._config import MailgunSourceConfig
+from products.warehouse_sources.sources.mailgun.mailgun import (
EVENTS_CONSISTENCY_LAG_SECONDS,
EVENTS_LOOKBACK_DAYS,
INVALID_KEY_ERROR,
@@ -46,12 +35,21 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.ma
sync_webhook_events,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.settings import (
+from products.warehouse_sources.sources.mailgun.settings import (
MAILGUN_ENDPOINTS,
WEBHOOK_EVENTS_ENDPOINT,
WEBHOOK_TYPES,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.source import MailgunSource
+from products.warehouse_sources.sources.mailgun.source import MailgunSource
+from products.warehouse_sources.sources.sdk import WebhookCreationResult
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import (
+ RecordedRequest,
+ ScriptedResponse,
+ SourceDriver,
+ always,
+ scripted_network,
+)
US_BASE = "https://api.mailgun.net"
POSTHOG_URL = "https://us.posthog.com/public/webhooks/abc"
@@ -232,9 +230,7 @@ class TestGetRows:
assert len(result.requests) == expected_requests
assert len(result.rows) == expected_rows
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.mailgun.MAX_PAGES_PER_CHAIN", 3
- )
+ @mock.patch("products.warehouse_sources.sources.mailgun.mailgun.MAX_PAGES_PER_CHAIN", 3)
def test_paging_stops_at_the_page_cap(self):
# A cursor that mints a fresh URL for every page never repeats, so the seen-URL guard can't
# see it and only the page cap ends the chain.
@@ -881,7 +877,7 @@ class TestWebhookManagement:
assert info.exists is False
assert "401" in (info.error or "")
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.mailgun.create_webhook")
+ @mock.patch("products.warehouse_sources.sources.mailgun.mailgun.create_webhook")
def test_sync_re_registers_so_new_domains_get_covered(self, mock_create):
mock_create.return_value = WebhookCreationResult(success=True)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun_source.py b/products/warehouse_sources/sources/mailgun/tests/test_mailgun_source.py
similarity index 82%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun_source.py
rename to products/warehouse_sources/sources/mailgun/tests/test_mailgun_source.py
index 4f137ef10c2e..3490585a85fc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun_source.py
+++ b/products/warehouse_sources/sources/mailgun/tests/test_mailgun_source.py
@@ -1,25 +1,15 @@
import pytest
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- RecordedRequest,
- ScriptedResponse,
- SourceDriver,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mailgun import (
- MailgunSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.settings import (
+from products.warehouse_sources.sources.mailgun._config import MailgunSourceConfig
+from products.warehouse_sources.sources.mailgun.settings import (
WEBHOOK_EVENTS_ENDPOINT,
WEBHOOK_RESOURCE_KEY,
WEBHOOK_TYPES,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.source import MailgunSource
+from products.warehouse_sources.sources.mailgun.source import MailgunSource
+from products.warehouse_sources.sources.sdk import ExternalWebhookInfo, WebhookCreationResult, WebhookDeletionResult
+from products.warehouse_sources.sources.sdk.testing import RecordedRequest, ScriptedResponse, SourceDriver
WEBHOOK_URL = "https://us.posthog.com/public/webhooks/abc"
@@ -37,9 +27,7 @@ class TestMailgunSource:
(False, "Mailgun rejected the API key."),
],
)
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.source.validate_mailgun_credentials"
- )
+ @mock.patch("products.warehouse_sources.sources.mailgun.source.validate_mailgun_credentials")
def test_validate_credentials(self, mock_validate, mock_return):
mock_validate.return_value = mock_return
@@ -78,7 +66,7 @@ class TestMailgunSource:
assert result.raised is None
assert result.paths == expected_paths
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.source.mailgun_source")
+ @mock.patch("products.warehouse_sources.sources.mailgun.source.mailgun_source")
def test_source_for_pipeline_plumbs_arguments(self, mock_mailgun_source):
inputs = mock.MagicMock()
inputs.schema_name = "events"
@@ -133,9 +121,7 @@ class TestMailgunSource:
def test_webhook_methods_target_the_configured_account_and_region(self, method, patched, return_value):
config = MailgunSourceConfig(api_key="key-123", region="eu")
- with mock.patch(
- f"products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.source.{patched}"
- ) as mock_fn:
+ with mock.patch(f"products.warehouse_sources.sources.mailgun.source.{patched}") as mock_fn:
mock_fn.return_value = return_value
result = getattr(self.source, method)(config, WEBHOOK_URL, self.team_id)
@@ -149,9 +135,7 @@ class TestMailgunSource:
(["events"], False),
],
)
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.source.sync_mailgun_webhook_events"
- )
+ @mock.patch("products.warehouse_sources.sources.mailgun.source.sync_mailgun_webhook_events")
def test_sync_only_touches_mailgun_when_the_webhook_table_is_selected(
self, mock_sync, eligible_schema_names, expect_call
):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun_webhook_template.py b/products/warehouse_sources/sources/mailgun/tests/test_mailgun_webhook_template.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun_webhook_template.py
rename to products/warehouse_sources/sources/mailgun/tests/test_mailgun_webhook_template.py
index 63edd56081da..ee31beba4046 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/tests/test_mailgun_webhook_template.py
+++ b/products/warehouse_sources/sources/mailgun/tests/test_mailgun_webhook_template.py
@@ -5,12 +5,9 @@ import hashlib
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.settings import (
- WEBHOOK_EVENT_NAMES,
- WEBHOOK_RESOURCE_KEY,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailgun.webhook_template import template
+from products.warehouse_sources.sources.mailgun.settings import WEBHOOK_EVENT_NAMES, WEBHOOK_RESOURCE_KEY
+from products.warehouse_sources.sources.mailgun.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
SOURCE_ID = "source_test_123"
SIGNING_KEY = "mailgun_http_webhook_signing_key"
@@ -27,7 +24,7 @@ def _event_data(event: str = "delivered") -> dict:
}
-class TestMailgunWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestMailgunWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/webhook_template.py b/products/warehouse_sources/sources/mailgun/webhook_template.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/webhook_template.py
rename to products/warehouse_sources/sources/mailgun/webhook_template.py
index 24022071635f..fe76e51d0adc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailgun/webhook_template.py
+++ b/products/warehouse_sources/sources/mailgun/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/sources/mailjet/__init__.py b/products/warehouse_sources/sources/mailjet/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mailjet.py b/products/warehouse_sources/sources/mailjet/_config.py
similarity index 73%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mailjet.py
rename to products/warehouse_sources/sources/mailjet/_config.py
index c22b0cfdff3f..cec10315179a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mailjet.py
+++ b/products/warehouse_sources/sources/mailjet/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/canonical_descriptions.py b/products/warehouse_sources/sources/mailjet/canonical_descriptions.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/canonical_descriptions.py
rename to products/warehouse_sources/sources/mailjet/canonical_descriptions.py
index 0d2bc1e9fd82..ec90c3b0bfd8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/mailjet/canonical_descriptions.py
@@ -5,9 +5,7 @@ Keyed by the resource names in `settings.py` `MAILJET_ENDPOINTS`, which match th
`ExternalDataSchema.name` of a synced Mailjet table. Columns absent here fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"contact": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/mailjet.py b/products/warehouse_sources/sources/mailjet/mailjet.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/mailjet.py
rename to products/warehouse_sources/sources/mailjet/mailjet.py
index f1dc5f3bea0d..71dec104bc56 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/mailjet.py
+++ b/products/warehouse_sources/sources/mailjet/mailjet.py
@@ -11,26 +11,7 @@ import structlog
from asgiref.sync import async_to_sync
from requests import Session
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- OffsetPaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.settings import (
+from products.warehouse_sources.sources.mailjet.settings import (
MAILJET_ENDPOINTS,
MAILJET_WEBHOOK_EVENTS,
STATCOUNTERS_EARLIEST_UNIX_TS,
@@ -41,6 +22,21 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.se
WEBHOOK_TABLE_NAME,
MailjetEndpointConfig,
)
+from products.warehouse_sources.sources.sdk import (
+ ExternalWebhookInfo,
+ OffsetPaginator,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SourceResponse,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSourceManager,
+ WebhookSyncResult,
+ make_tracked_session,
+ rest_api_resource,
+ validate_via_probe,
+)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
MAILJET_BASE_URL = "https://api.mailjet.com/v3/REST"
WEBHOOK_PATH = "/eventcallbackurl"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/settings.py b/products/warehouse_sources/sources/mailjet/settings.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/settings.py
rename to products/warehouse_sources/sources/mailjet/settings.py
index 60d06bb7f843..d273e4951880 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/settings.py
+++ b/products/warehouse_sources/sources/mailjet/settings.py
@@ -1,7 +1,7 @@
from dataclasses import dataclass, field
from typing import Optional
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType
@dataclass
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/source.py b/products/warehouse_sources/sources/mailjet/source.py
similarity index 82%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/source.py
rename to products/warehouse_sources/sources/mailjet/source.py
index a8711712aab7..86f357ec7051 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/source.py
+++ b/products/warehouse_sources/sources/mailjet/source.py
@@ -1,53 +1,46 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.mailjet import mailjet as api_client
+from products.warehouse_sources.sources.mailjet._config import MailjetSourceConfig
+from products.warehouse_sources.sources.mailjet.mailjet import (
+ MailjetResumeConfig,
+ mailjet_source,
+ validate_credentials as validate_mailjet_credentials,
+)
+from products.warehouse_sources.sources.mailjet.settings import (
+ ENDPOINTS,
+ INCREMENTAL_FIELDS,
+ MAILJET_WEBHOOK_EVENTS,
+ SCHEMA_TO_WEBHOOK_RESOURCE,
+ WEBHOOK_SCHEMA_NAMES,
+)
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
build_endpoint_schemas,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mailjet import (
- MailjetSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet import mailjet as api_client
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.mailjet import (
- MailjetResumeConfig,
- mailjet_source,
- validate_credentials as validate_mailjet_credentials,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.settings import (
- ENDPOINTS,
- INCREMENTAL_FIELDS,
- MAILJET_WEBHOOK_EVENTS,
- SCHEMA_TO_WEBHOOK_RESOURCE,
- WEBHOOK_SCHEMA_NAMES,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@SourceRegistry.register
@@ -129,9 +122,7 @@ You can find your API key and secret key in your [Mailjet API key management pag
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.mailjet.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
@@ -182,9 +173,7 @@ You can find your API key and secret key in your [Mailjet API key management pag
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.webhook_template import ( # noqa: PLC0415
- template,
- )
+ from products.warehouse_sources.sources.mailjet.webhook_template import template # noqa: PLC0415
return template
diff --git a/products/warehouse_sources/sources/mailjet/tests/__init__.py b/products/warehouse_sources/sources/mailjet/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/tests/test_mailjet.py b/products/warehouse_sources/sources/mailjet/tests/test_mailjet.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/tests/test_mailjet.py
rename to products/warehouse_sources/sources/mailjet/tests/test_mailjet.py
index 5d9a6ba9f108..4b1da3553495 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/tests/test_mailjet.py
+++ b/products/warehouse_sources/sources/mailjet/tests/test_mailjet.py
@@ -7,16 +7,8 @@ from unittest import mock
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
- always,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mailjet import (
- MailjetSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.mailjet import (
+from products.warehouse_sources.sources.mailjet._config import MailjetSourceConfig
+from products.warehouse_sources.sources.mailjet.mailjet import (
MAILJET_BASE_URL,
WEBHOOK_PATH,
MailjetResumeConfig,
@@ -30,14 +22,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.ma
sync_webhook_events,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.settings import (
+from products.warehouse_sources.sources.mailjet.settings import (
MAILJET_ENDPOINTS,
MAILJET_WEBHOOK_EVENTS,
STATCOUNTERS_EARLIEST_UNIX_TS,
STATCOUNTERS_ENDPOINT,
WEBHOOK_TABLE_NAME,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.source import MailJetSource
+from products.warehouse_sources.sources.mailjet.source import MailJetSource
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network
def _response(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/tests/test_mailjet_source.py b/products/warehouse_sources/sources/mailjet/tests/test_mailjet_source.py
similarity index 81%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/tests/test_mailjet_source.py
rename to products/warehouse_sources/sources/mailjet/tests/test_mailjet_source.py
index 80b56c44fb01..09985ac5dbc6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/tests/test_mailjet_source.py
+++ b/products/warehouse_sources/sources/mailjet/tests/test_mailjet_source.py
@@ -1,20 +1,18 @@
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mailjet import (
- MailjetSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.settings import (
+from products.warehouse_sources.sources.mailjet._config import MailjetSourceConfig
+from products.warehouse_sources.sources.mailjet.settings import (
MAILJET_WEBHOOK_EVENTS,
SCHEMA_TO_WEBHOOK_RESOURCE,
WEBHOOK_SCHEMA_NAMES,
WEBHOOK_TABLE_NAME,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.source import MailJetSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.webhook_template import template
+from products.warehouse_sources.sources.mailjet.source import MailJetSource
+from products.warehouse_sources.sources.mailjet.webhook_template import template
+from products.warehouse_sources.sources.sdk import WebhookSourceManager
WEBHOOK_URL = "https://webhooks.us.posthog.com/public/webhooks/dwh/hog-fn-1"
-API_CLIENT_PATCH = "products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.source.api_client"
+API_CLIENT_PATCH = "products.warehouse_sources.sources.mailjet.source.api_client"
class TestMailJetSource:
@@ -67,9 +65,7 @@ class TestMailJetSource:
self.source.sync_webhook_events(self.config, WEBHOOK_URL, self.team_id, [WEBHOOK_TABLE_NAME])
mock_sync.assert_called_once_with("key", "secret", WEBHOOK_URL, list(MAILJET_WEBHOOK_EVENTS))
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.source.validate_mailjet_credentials"
- )
+ @mock.patch("products.warehouse_sources.sources.mailjet.source.validate_mailjet_credentials")
def test_validate_credentials_success(self, mock_validate):
mock_validate.return_value = True
@@ -79,9 +75,7 @@ class TestMailJetSource:
assert error_message is None
mock_validate.assert_called_once_with(self.config.api_key, self.config.secret_key)
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.source.validate_mailjet_credentials"
- )
+ @mock.patch("products.warehouse_sources.sources.mailjet.source.validate_mailjet_credentials")
def test_validate_credentials_failure(self, mock_validate):
mock_validate.return_value = False
@@ -90,7 +84,7 @@ class TestMailJetSource:
assert is_valid is False
assert error_message == "Invalid Mailjet API key or secret key"
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.source.mailjet_source")
+ @mock.patch("products.warehouse_sources.sources.mailjet.source.mailjet_source")
def test_source_for_pipeline(self, mock_mailjet_source):
mock_mailjet_source.return_value = mock.MagicMock()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/tests/test_mailjet_webhook_template.py b/products/warehouse_sources/sources/mailjet/tests/test_mailjet_webhook_template.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/tests/test_mailjet_webhook_template.py
rename to products/warehouse_sources/sources/mailjet/tests/test_mailjet_webhook_template.py
index c1839294b330..5cc1c5ef595b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/tests/test_mailjet_webhook_template.py
+++ b/products/warehouse_sources/sources/mailjet/tests/test_mailjet_webhook_template.py
@@ -4,16 +4,16 @@ import hashlib
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.settings import MAILJET_WEBHOOK_EVENTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.mailjet.webhook_template import template
+from products.warehouse_sources.sources.mailjet.settings import MAILJET_WEBHOOK_EVENTS
+from products.warehouse_sources.sources.mailjet.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
SOURCE_ID = "source_test_123"
AUTHORIZATION_HEADER = "Basic " + base64.b64encode(b"posthog:s3cr3t").decode()
SCHEMA_MAPPING = {"messageevent": "schema_message_events"}
-class TestMailjetWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestMailjetWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/webhook_template.py b/products/warehouse_sources/sources/mailjet/webhook_template.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/webhook_template.py
rename to products/warehouse_sources/sources/mailjet/webhook_template.py
index eb57b7533e72..cea5b9fed708 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mailjet/webhook_template.py
+++ b/products/warehouse_sources/sources/mailjet/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/sources/mem0/__init__.py b/products/warehouse_sources/sources/mem0/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mem0.py b/products/warehouse_sources/sources/mem0/_config.py
similarity index 76%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mem0.py
rename to products/warehouse_sources/sources/mem0/_config.py
index 7750e2dc3609..916db4f3518c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/mem0.py
+++ b/products/warehouse_sources/sources/mem0/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/canonical_descriptions.py b/products/warehouse_sources/sources/mem0/canonical_descriptions.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mem0/canonical_descriptions.py
rename to products/warehouse_sources/sources/mem0/canonical_descriptions.py
index 26485d32c00f..0adad59ff402 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/mem0/canonical_descriptions.py
@@ -5,9 +5,7 @@ Keyed by the endpoint names in `settings.py` `MEM0_ENDPOINTS`, which match the
`ExternalDataSchema.name` of a synced Mem0 table. Columns absent here fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"memories": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/mem0.py b/products/warehouse_sources/sources/mem0/mem0.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mem0/mem0.py
rename to products/warehouse_sources/sources/mem0/mem0.py
index 972e1e7ad49f..132f5f9c295c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/mem0.py
+++ b/products/warehouse_sources/sources/mem0/mem0.py
@@ -17,28 +17,7 @@ from urllib.parse import urljoin, urlparse
import structlog
from requests.exceptions import HTTPError
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
- rest_api_resources,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
- rename_parent_fields,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- JSONResponsePaginator,
- SinglePagePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- ClientConfig,
- EndpointResource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.mem0.settings import (
+from products.warehouse_sources.sources.mem0.settings import (
ENTITIES_ENDPOINT,
EVENTS_ENDPOINT,
MEM0_BASE_URL,
@@ -48,6 +27,21 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mem0.setti
ORGANIZATIONS_ENDPOINT,
PROJECTS_ENDPOINT,
)
+from products.warehouse_sources.sources.sdk import (
+ ClientConfig,
+ EndpointResource,
+ JSONResponsePaginator,
+ Resource,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SinglePagePaginator,
+ SourceResponse,
+ make_tracked_session,
+ rename_parent_fields,
+ rest_api_resource,
+ rest_api_resources,
+ validate_via_probe,
+)
logger = structlog.get_logger(__name__)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/settings.py b/products/warehouse_sources/sources/mem0/settings.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mem0/settings.py
rename to products/warehouse_sources/sources/mem0/settings.py
index c0631109f63b..43472040fcbd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/settings.py
+++ b/products/warehouse_sources/sources/mem0/settings.py
@@ -20,8 +20,7 @@ Reference: https://docs.mem0.ai/api-reference
from dataclasses import dataclass, field
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import UNVERSIONED_API_VERSION
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import UNVERSIONED_API_VERSION, IncrementalField, IncrementalFieldType
MEM0_BASE_URL = "https://api.mem0.ai"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/source.py b/products/warehouse_sources/sources/mem0/source.py
similarity index 84%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mem0/source.py
rename to products/warehouse_sources/sources/mem0/source.py
index 6c5338b0c827..e007258e867a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/source.py
+++ b/products/warehouse_sources/sources/mem0/source.py
@@ -1,40 +1,37 @@
from typing import Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- UNVERSIONED_API_VERSION,
- FieldType,
- ResumableSource,
- VersionDeprecation,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mem0 import Mem0SourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.mem0.mem0 import (
+from products.warehouse_sources.sources.mem0._config import Mem0SourceConfig
+from products.warehouse_sources.sources.mem0.mem0 import (
Mem0ResumeConfig,
mem0_source,
validate_credentials as validate_mem0_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mem0.settings import (
+from products.warehouse_sources.sources.mem0.settings import (
DEFAULT_VERSION,
ENDPOINTS,
INCREMENTAL_FIELDS,
MEM0_ENDPOINTS,
SUPPORTED_VERSIONS,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.sdk import (
+ UNVERSIONED_API_VERSION,
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ FieldType,
+ ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
+ VersionDeprecation,
+ detect,
+)
# Mem0 can push memory:add/update/delete/categorize webhooks, but we deliberately don't wire a
# WebhookSource here: deliveries carry only {event_details: {id, data.memory, event}} — no
@@ -111,9 +108,7 @@ You can find your API key in the [Mem0 dashboard](https://app.mem0.ai/dashboard/
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.mem0.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.mem0.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/mem0/tests/__init__.py b/products/warehouse_sources/sources/mem0/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/tests/test_mem0.py b/products/warehouse_sources/sources/mem0/tests/test_mem0.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mem0/tests/test_mem0.py
rename to products/warehouse_sources/sources/mem0/tests/test_mem0.py
index b94b045cb823..f51ec9d7905e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/tests/test_mem0.py
+++ b/products/warehouse_sources/sources/mem0/tests/test_mem0.py
@@ -4,19 +4,10 @@ from typing import Any
import requests
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mem0 import Mem0SourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.mem0 import mem0 as api_client
-from products.warehouse_sources.backend.temporal.data_imports.sources.mem0.mem0 import (
- _MATCH_ALL_FILTER,
- Mem0ResumeConfig,
- validate_credentials,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mem0.settings import (
+from products.warehouse_sources.sources.mem0 import mem0 as api_client
+from products.warehouse_sources.sources.mem0._config import Mem0SourceConfig
+from products.warehouse_sources.sources.mem0.mem0 import _MATCH_ALL_FILTER, Mem0ResumeConfig, validate_credentials
+from products.warehouse_sources.sources.mem0.settings import (
ENTITIES_ENDPOINT,
EVENTS_ENDPOINT,
MEM0_BASE_URL,
@@ -24,7 +15,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.mem0.setti
MEMORY_HISTORY_ENDPOINT,
PROJECTS_ENDPOINT,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mem0.source import Mem0Source
+from products.warehouse_sources.sources.mem0.source import Mem0Source
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, scripted_network
def _driver(org_id: str | None = None, project_id: str | None = None) -> SourceDriver:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/tests/test_mem0_source.py b/products/warehouse_sources/sources/mem0/tests/test_mem0_source.py
similarity index 76%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/mem0/tests/test_mem0_source.py
rename to products/warehouse_sources/sources/mem0/tests/test_mem0_source.py
index 337e435fa339..87d582126d4f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/mem0/tests/test_mem0_source.py
+++ b/products/warehouse_sources/sources/mem0/tests/test_mem0_source.py
@@ -1,18 +1,11 @@
from unittest.mock import MagicMock, patch
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.mem0 import Mem0SourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.mem0.settings import (
- ENTITIES_ENDPOINT,
- EVENTS_ENDPOINT,
- MEMORIES_ENDPOINT,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.mem0.source import Mem0Source
-
-_SOURCE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.mem0.source"
+from products.warehouse_sources.sources.mem0._config import Mem0SourceConfig
+from products.warehouse_sources.sources.mem0.settings import ENTITIES_ENDPOINT, EVENTS_ENDPOINT, MEMORIES_ENDPOINT
+from products.warehouse_sources.sources.mem0.source import Mem0Source
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, scripted_network
+
+_SOURCE_MODULE = "products.warehouse_sources.sources.mem0.source"
def _config(api_key: str = "m0-test", org_id: str | None = None, project_id: str | None = None) -> Mem0SourceConfig:
diff --git a/products/warehouse_sources/sources/particle/__init__.py b/products/warehouse_sources/sources/particle/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/particle.py b/products/warehouse_sources/sources/particle/_config.py
similarity index 76%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/particle.py
rename to products/warehouse_sources/sources/particle/_config.py
index 2d13bc94b90b..7ed290bd1551 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/particle.py
+++ b/products/warehouse_sources/sources/particle/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/canonical_descriptions.py b/products/warehouse_sources/sources/particle/canonical_descriptions.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/particle/canonical_descriptions.py
rename to products/warehouse_sources/sources/particle/canonical_descriptions.py
index 4f19ff12fba5..9e78385fc787 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/particle/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"devices": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/particle.py b/products/warehouse_sources/sources/particle/particle.py
similarity index 92%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/particle/particle.py
rename to products/warehouse_sources/sources/particle/particle.py
index 38c390cf4d62..b560c44831e4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/particle.py
+++ b/products/warehouse_sources/sources/particle/particle.py
@@ -21,26 +21,8 @@ from typing import Any, cast
import pyarrow as pa
from requests import Response, Session
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- Endpoint,
- PaginatorConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.particle import (
- ParticleSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.particle.settings import (
+from products.warehouse_sources.sources.particle._config import ParticleSourceConfig
+from products.warehouse_sources.sources.particle.settings import (
DEVICES_PAGE_SIZE,
DEVICES_SCHEMA_NAME,
PARTICLE_BASE_URL,
@@ -48,6 +30,18 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.particle.s
WEBHOOK_QUERY,
WEBHOOK_SECRET_HEADER,
)
+from products.warehouse_sources.sources.sdk import (
+ Endpoint,
+ ExternalWebhookInfo,
+ PaginatorConfig,
+ RESTAPIConfig,
+ SourceResponse,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ make_tracked_session,
+ rest_api_resource,
+)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
logger = logging.getLogger(__name__)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/settings.py b/products/warehouse_sources/sources/particle/settings.py
similarity index 100%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/particle/settings.py
rename to products/warehouse_sources/sources/particle/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/source.py b/products/warehouse_sources/sources/particle/source.py
similarity index 86%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/particle/source.py
rename to products/warehouse_sources/sources/particle/source.py
index c80bcf6aa4cc..101a79a92776 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/source.py
+++ b/products/warehouse_sources/sources/particle/source.py
@@ -3,33 +3,8 @@ from typing import TYPE_CHECKING, Any, Optional, cast
from asgiref.sync import async_to_sync
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- SimpleSource,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.particle import (
- ParticleSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.particle.particle import (
+from products.warehouse_sources.sources.particle._config import ParticleSourceConfig
+from products.warehouse_sources.sources.particle.particle import (
_webhook_table_transformer,
create_webhook as create_particle_webhook,
delete_webhook as delete_particle_webhook,
@@ -37,17 +12,37 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.particle.p
particle_source,
probe_devices,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.particle.settings import (
+from products.warehouse_sources.sources.particle.settings import (
DEVICES_SCHEMA_NAME,
ENDPOINTS,
EVENTS_SCHEMA_NAME,
WEBHOOK_RESOURCE_MAP,
WEBHOOK_SCHEMA_NAMES,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
+ ReleaseStatus,
+ SimpleSource,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSource,
+ WebhookSourceManager,
+ detect,
+)
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@SourceRegistry.register
@@ -70,9 +65,7 @@ class ParticleSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.particle.webhook_template import ( # noqa: PLC0415
- template,
- )
+ from products.warehouse_sources.sources.particle.webhook_template import template # noqa: PLC0415
return template
@@ -154,7 +147,7 @@ For **sandbox devices**, Particle has no API users, so use a personal access tok
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.particle.canonical_descriptions import ( # noqa: PLC0415
+ from products.warehouse_sources.sources.particle.canonical_descriptions import ( # noqa: PLC0415
CANONICAL_DESCRIPTIONS,
)
diff --git a/products/warehouse_sources/sources/particle/tests/__init__.py b/products/warehouse_sources/sources/particle/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/tests/test_particle.py b/products/warehouse_sources/sources/particle/tests/test_particle.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/particle/tests/test_particle.py
rename to products/warehouse_sources/sources/particle/tests/test_particle.py
index 16bb0b94dc32..576559a75572 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/tests/test_particle.py
+++ b/products/warehouse_sources/sources/particle/tests/test_particle.py
@@ -2,18 +2,8 @@ from typing import Any
import pytest
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
- always,
- route,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.particle import (
- ParticleSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.particle.particle import (
+from products.warehouse_sources.sources.particle._config import ParticleSourceConfig
+from products.warehouse_sources.sources.particle.particle import (
WEBHOOK_AUTH_ERROR,
_webhook_table_transformer,
create_webhook,
@@ -21,8 +11,16 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.particle.p
event_prefixes,
get_external_webhook_info,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.particle.settings import WEBHOOK_SECRET_HEADER
-from products.warehouse_sources.backend.temporal.data_imports.sources.particle.source import ParticleSource
+from products.warehouse_sources.sources.particle.settings import WEBHOOK_SECRET_HEADER
+from products.warehouse_sources.sources.particle.source import ParticleSource
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import (
+ ScriptedResponse,
+ SourceDriver,
+ always,
+ route,
+ scripted_network,
+)
WEBHOOK_URL = "https://webhooks.us.posthog.com/public/webhooks/dwh/0199f0c4-1b2a-7c3d-8e4f-5a6b7c8d9e0f"
TOKEN = "particle-api-user-token"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/tests/test_particle_webhook_template.py b/products/warehouse_sources/sources/particle/tests/test_particle_webhook_template.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/particle/tests/test_particle_webhook_template.py
rename to products/warehouse_sources/sources/particle/tests/test_particle_webhook_template.py
index 957c960a2309..141828528fd4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/tests/test_particle_webhook_template.py
+++ b/products/warehouse_sources/sources/particle/tests/test_particle_webhook_template.py
@@ -2,16 +2,16 @@ import json
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.particle.settings import WEBHOOK_SECRET_HEADER
-from products.warehouse_sources.backend.temporal.data_imports.sources.particle.webhook_template import template
+from products.warehouse_sources.sources.particle.settings import WEBHOOK_SECRET_HEADER
+from products.warehouse_sources.sources.particle.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
WEBHOOK_SECRET = "c2VjcmV0LXNoYXJlZC13aXRoLXBhcnRpY2xl"
SCHEMA_MAPPING = {"events": "schema_events"}
DEVICE_ID = "e00fce68f1d2c3b4a5968778"
-class TestParticleWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestParticleWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/webhook_template.py b/products/warehouse_sources/sources/particle/webhook_template.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/particle/webhook_template.py
rename to products/warehouse_sources/sources/particle/webhook_template.py
index 85b32debf516..2a9c8b2a7208 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/particle/webhook_template.py
+++ b/products/warehouse_sources/sources/particle/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
# Particle doesn't sign webhook deliveries. `create_webhook` pins a generated secret in the
# x-posthog-webhook-secret header of every hook it creates, and this template rejects a delivery
diff --git a/products/warehouse_sources/sources/paymongo/__init__.py b/products/warehouse_sources/sources/paymongo/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/paymongo.py b/products/warehouse_sources/sources/paymongo/_config.py
similarity index 71%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/paymongo.py
rename to products/warehouse_sources/sources/paymongo/_config.py
index ea05c7f87a7c..0509ab449ee7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/paymongo.py
+++ b/products/warehouse_sources/sources/paymongo/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/canonical_descriptions.py b/products/warehouse_sources/sources/paymongo/canonical_descriptions.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/canonical_descriptions.py
rename to products/warehouse_sources/sources/paymongo/canonical_descriptions.py
index 9893db7e1951..e18e15125d73 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/paymongo/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"payments": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/paymongo.py b/products/warehouse_sources/sources/paymongo/paymongo.py
similarity index 89%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/paymongo.py
rename to products/warehouse_sources/sources/paymongo/paymongo.py
index 7d247e4f72a5..021987747182 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/paymongo.py
+++ b/products/warehouse_sources/sources/paymongo/paymongo.py
@@ -7,43 +7,32 @@ from urllib.parse import quote
from requests import Response
from requests.exceptions import HTTPError
-from posthog.dataclasses import frozen
-
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.paymongo.settings import BASE_URL, ENDPOINTS, PAGE_SIZE, WEBHOOK_EVENTS
+from products.warehouse_sources.sources.sdk import (
+ Endpoint,
+ EndpointResource,
ExternalWebhookInfo,
+ HttpBasicAuth,
+ JSONResponseCursorPaginator,
+ RESTAPIConfig,
+ RESTClient,
+ ResumableSourceManager,
+ SourceInputs,
+ SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
+ WebhookSourceManager,
+ frozen,
+ make_tracked_session,
rest_api_resource,
rest_api_resources,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import HttpBasicAuth
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- JSONResponseCursorPaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- Endpoint,
- EndpointResource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import schema_for_resource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.paymongo.settings import (
- BASE_URL,
- ENDPOINTS,
- PAGE_SIZE,
- WEBHOOK_EVENTS,
+ schema_for_resource,
)
if TYPE_CHECKING:
import pyarrow as pa
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@frozen
@@ -169,7 +158,7 @@ def pull_rows(
def webhook_table(table: "pa.Table", client: "PaymongoClient") -> "pa.Table":
- from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import ( # noqa: PLC0415 - keeps Arrow off the source registration path
+ from products.warehouse_sources.sources.sdk.internals import ( # noqa: PLC0415 - keeps Arrow off the source registration path
table_from_py_list,
)
@@ -334,7 +323,7 @@ class PaymongoClient:
def webhook_template() -> "HogFunctionTemplateDC":
- from products.cdp.backend.templates.hog_function_template import ( # noqa: PLC0415 - keeps the template dependencies off the source registration path
+ from products.warehouse_sources.sources.sdk import ( # noqa: PLC0415 - keeps the template dependencies off the source registration path
HogFunctionTemplateDC,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/settings.py b/products/warehouse_sources/sources/paymongo/settings.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/settings.py
rename to products/warehouse_sources/sources/paymongo/settings.py
index d0bf8b431bcb..32275dcaa9fe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/settings.py
+++ b/products/warehouse_sources/sources/paymongo/settings.py
@@ -1,8 +1,6 @@
from typing import Literal
-from posthog.dataclasses import frozen
-
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.sources.sdk import IncrementalField, frozen
BASE_URL = "https://api.paymongo.com/v1/"
INVOICES_BASE_URL = "https://invoices-api.paymongo.com/v1/"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/source.py b/products/warehouse_sources/sources/paymongo/source.py
similarity index 81%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/source.py
rename to products/warehouse_sources/sources/paymongo/source.py
index 3af42b47ace8..f48e8dbf2706 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/source.py
+++ b/products/warehouse_sources/sources/paymongo/source.py
@@ -2,53 +2,41 @@ from typing import TYPE_CHECKING, cast
from requests.exceptions import HTTPError
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.paymongo._config import PaymongoSourceConfig
+from products.warehouse_sources.sources.paymongo.canonical_descriptions import CANONICAL_DESCRIPTIONS
+from products.warehouse_sources.sources.paymongo.paymongo import (
+ PaymongoClient,
+ PaymongoResumeConfig,
+ paymongo_source,
+ webhook_template,
+)
+from products.warehouse_sources.sources.paymongo.settings import ENDPOINTS, INCREMENTAL_FIELDS
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
+ WebhookSourceManager,
build_endpoint_schemas,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.paymongo import (
- PaymongoSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.paymongo.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.paymongo.paymongo import (
- PaymongoClient,
- PaymongoResumeConfig,
- paymongo_source,
- webhook_template,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.paymongo.settings import (
- ENDPOINTS,
- INCREMENTAL_FIELDS,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
def webhook_error(error: HTTPError) -> str:
diff --git a/products/warehouse_sources/sources/paymongo/tests/__init__.py b/products/warehouse_sources/sources/paymongo/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.py b/products/warehouse_sources/sources/paymongo/tests/test_paymongo.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.py
rename to products/warehouse_sources/sources/paymongo/tests/test_paymongo.py
index e2362ed02a06..65a90efb8d0e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo.py
+++ b/products/warehouse_sources/sources/paymongo/tests/test_paymongo.py
@@ -13,25 +13,18 @@ import pyarrow as pa
from requests import PreparedRequest, Response
from requests.exceptions import HTTPError
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.paymongo import (
- PaymongoSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.paymongo.paymongo import (
+from products.warehouse_sources.sources.paymongo._config import PaymongoSourceConfig
+from products.warehouse_sources.sources.paymongo.paymongo import (
PaymongoClient,
PaymongoResumeConfig,
paymongo_source,
pull_rows,
webhook_table,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.paymongo.source import PaymongoSource
+from products.warehouse_sources.sources.paymongo.source import PaymongoSource
+from products.warehouse_sources.sources.sdk import ResumableSourceManager, SourceInputs
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, scripted_network
def source_inputs(name: str = "payments") -> SourceInputs:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo_source.py b/products/warehouse_sources/sources/paymongo/tests/test_paymongo_source.py
similarity index 86%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo_source.py
rename to products/warehouse_sources/sources/paymongo/tests/test_paymongo_source.py
index 98a4b36b2200..4a2084ace765 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/paymongo/tests/test_paymongo_source.py
+++ b/products/warehouse_sources/sources/paymongo/tests/test_paymongo_source.py
@@ -7,18 +7,15 @@ from typing import Any
import pytest
from unittest.mock import MagicMock
-from products.cdp.backend.validation import compile_hog
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import UnknownResourceError
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+from products.warehouse_sources.sources.paymongo._config import PaymongoSourceConfig
+from products.warehouse_sources.sources.paymongo.source import PaymongoSource
+from products.warehouse_sources.sources.sdk import UnknownResourceError
+from products.warehouse_sources.sources.sdk.testing import (
ScriptedResponse,
+ compile_hog,
+ execute_bytecode,
scripted_network,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.paymongo import (
- PaymongoSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.paymongo.source import PaymongoSource
-
-from common.hogvm.python.execute import execute_bytecode
@pytest.mark.parametrize("operation", ["create_webhook", "delete_webhook"])
diff --git a/products/warehouse_sources/sources/pipedrive/__init__.py b/products/warehouse_sources/sources/pipedrive/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/pipedrive.py b/products/warehouse_sources/sources/pipedrive/_config.py
similarity index 74%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/pipedrive.py
rename to products/warehouse_sources/sources/pipedrive/_config.py
index 98749ebff464..e71625a9811d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/pipedrive.py
+++ b/products/warehouse_sources/sources/pipedrive/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/api_inventory.md b/products/warehouse_sources/sources/pipedrive/api_inventory.md
similarity index 100%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/api_inventory.md
rename to products/warehouse_sources/sources/pipedrive/api_inventory.md
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/canonical_descriptions.py b/products/warehouse_sources/sources/pipedrive/canonical_descriptions.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/canonical_descriptions.py
rename to products/warehouse_sources/sources/pipedrive/canonical_descriptions.py
index 98cd1781249f..ba43d98300c4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/pipedrive/canonical_descriptions.py
@@ -6,9 +6,7 @@ Keyed by the endpoint names in `settings.py` (`ENDPOINTS`), which match the
enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"deals": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/pipedrive.py b/products/warehouse_sources/sources/pipedrive/pipedrive.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/pipedrive.py
rename to products/warehouse_sources/sources/pipedrive/pipedrive.py
index 331dc740da1c..1baa37cbb1f8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/pipedrive.py
+++ b/products/warehouse_sources/sources/pipedrive/pipedrive.py
@@ -14,34 +14,7 @@ from dateutil import parser as dateutil_parser
from requests import Session
from requests.exceptions import HTTPError, RequestException
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import APIKeyAuth
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.config_setup import (
- create_response_hooks,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- BasePaginator,
- JSONResponseCursorPaginator,
- OffsetPaginator,
- SinglePagePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client import RESTClient
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import ClientConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.settings import (
+from products.warehouse_sources.sources.pipedrive.settings import (
DEAL_IDS_PER_REQUEST,
DEAL_PARENT_PATHS,
MAIL_FOLDERS,
@@ -51,6 +24,27 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.
PipedriveEndpointConfig,
endpoints_for_version,
)
+from products.warehouse_sources.sources.sdk import (
+ APIKeyAuth,
+ BasePaginator,
+ ClientConfig,
+ ExternalWebhookInfo,
+ JSONResponseCursorPaginator,
+ OffsetPaginator,
+ RESTAPIConfig,
+ RESTClient,
+ ResumableSourceManager,
+ SinglePagePaginator,
+ SourceResponse,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSourceManager,
+ create_response_hooks,
+ make_tracked_session,
+ rest_api_resource,
+ validate_via_probe,
+)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
LOGGER = structlog.get_logger(__name__)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/settings.py b/products/warehouse_sources/sources/pipedrive/settings.py
similarity index 100%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/settings.py
rename to products/warehouse_sources/sources/pipedrive/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/source.py b/products/warehouse_sources/sources/pipedrive/source.py
similarity index 86%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/source.py
rename to products/warehouse_sources/sources/pipedrive/source.py
index cd4bda20324e..9cd21389f744 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/source.py
+++ b/products/warehouse_sources/sources/pipedrive/source.py
@@ -1,35 +1,8 @@
import datetime
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
- DataWarehouseSourceCategory,
- ReleaseStatus,
- SourceConfig,
- SourceFieldInputConfig,
- SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
- VersionDeprecation,
- WebhookCreationResult,
- WebhookDeletionResult,
- WebhookSource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.pipedrive import (
- PipedriveSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.pipedrive import (
+from products.warehouse_sources.sources.pipedrive._config import PipedriveSourceConfig
+from products.warehouse_sources.sources.pipedrive.pipedrive import (
PipedriveResumeConfig,
create_webhook as create_pipedrive_webhook,
delete_webhook as delete_pipedrive_webhook,
@@ -38,15 +11,37 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.
pipedrive_source,
validate_credentials as validate_pipedrive_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.settings import (
+from products.warehouse_sources.sources.pipedrive.settings import (
ENDPOINTS,
WEBHOOK_ENTITY_BY_SCHEMA,
webhook_schema_names,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
+ DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
+ ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
+ SourceConfig,
+ SourceFieldInputConfig,
+ SourceFieldInputConfigType,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
+ VersionDeprecation,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSource,
+ WebhookSourceManager,
+ detect,
+)
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@SourceRegistry.register
@@ -83,7 +78,7 @@ class PipedriveSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.webhook_template import template
+ from products.warehouse_sources.sources.pipedrive.webhook_template import template
return template
@@ -174,9 +169,7 @@ Deletions in Pipedrive are not applied to tables synced by webhook. Switch a tab
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.pipedrive.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/pipedrive/tests/__init__.py b/products/warehouse_sources/sources/pipedrive/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/tests/test_pipedrive.py b/products/warehouse_sources/sources/pipedrive/tests/test_pipedrive.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/tests/test_pipedrive.py
rename to products/warehouse_sources/sources/pipedrive/tests/test_pipedrive.py
index 30d841991e99..fd6edc11a334 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/tests/test_pipedrive.py
+++ b/products/warehouse_sources/sources/pipedrive/tests/test_pipedrive.py
@@ -6,13 +6,7 @@ from unittest import mock
from requests import Response
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- always,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.pipedrive import (
+from products.warehouse_sources.sources.pipedrive.pipedrive import (
PAGE_SIZE,
WEBHOOK_AUTH_USER,
PipedriveResumeConfig,
@@ -25,7 +19,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.
pipedrive_source,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.settings import endpoints_for_version
+from products.warehouse_sources.sources.pipedrive.settings import endpoints_for_version
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, always, scripted_network
# RESTClient builds its session via make_tracked_session in the rest_client module.
CLIENT_SESSION_PATCH = "products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client.make_tracked_session"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/tests/test_pipedrive_source.py b/products/warehouse_sources/sources/pipedrive/tests/test_pipedrive_source.py
similarity index 85%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/tests/test_pipedrive_source.py
rename to products/warehouse_sources/sources/pipedrive/tests/test_pipedrive_source.py
index eadd1698a66b..44f019971505 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/tests/test_pipedrive_source.py
+++ b/products/warehouse_sources/sources/pipedrive/tests/test_pipedrive_source.py
@@ -3,13 +3,10 @@ import datetime
import pytest
from unittest import mock
-from products.warehouse_sources.backend.facade.source_config import SourceFieldInputConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.pipedrive import (
- PipedriveSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.settings import ENDPOINTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.source import PipedriveSource
+from products.warehouse_sources.sources.pipedrive._config import PipedriveSourceConfig
+from products.warehouse_sources.sources.pipedrive.settings import ENDPOINTS
+from products.warehouse_sources.sources.pipedrive.source import PipedriveSource
+from products.warehouse_sources.sources.sdk import SourceFieldInputConfig, WebhookSourceManager
class TestPipedriveSource:
@@ -52,9 +49,7 @@ class TestPipedriveSource:
(None, None, False, "Couldn't validate your Pipedrive credentials"),
],
)
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.source.validate_pipedrive_credentials"
- )
+ @mock.patch("products.warehouse_sources.sources.pipedrive.source.validate_pipedrive_credentials")
def test_validate_credentials(
self,
mock_validate: mock.MagicMock,
@@ -74,9 +69,7 @@ class TestPipedriveSource:
assert message is not None and expected_message_substring in message
mock_validate.assert_called_once_with("acme", "token")
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.source.validate_pipedrive_credentials"
- )
+ @mock.patch("products.warehouse_sources.sources.pipedrive.source.validate_pipedrive_credentials")
def test_validate_credentials_rejects_invalid_domain(self, mock_validate: mock.MagicMock) -> None:
mock_validate.side_effect = ValueError("Invalid Pipedrive company domain: 'evil.com'")
is_valid, message = self.source.validate_credentials(
@@ -94,7 +87,7 @@ class TestPipedriveSource:
(None, "v2"),
],
)
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.source.pipedrive_source")
+ @mock.patch("products.warehouse_sources.sources.pipedrive.source.pipedrive_source")
def test_source_for_pipeline_plumbs_arguments(
self, mock_pipedrive_source: mock.MagicMock, pinned_version: str | None, expected_version: str
) -> None:
@@ -113,7 +106,7 @@ class TestPipedriveSource:
assert kwargs["api_version"] == expected_version
assert kwargs["resumable_source_manager"] is manager
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.source.pipedrive_source")
+ @mock.patch("products.warehouse_sources.sources.pipedrive.source.pipedrive_source")
def test_source_for_pipeline_normalizes_company_domain(self, mock_pipedrive_source: mock.MagicMock) -> None:
inputs = mock.MagicMock()
inputs.schema_name = "deals"
@@ -182,14 +175,12 @@ class TestPipedriveSource:
],
)
def test_webhook_management_passes_domain_then_token(self, method_name: str, patched: str) -> None:
- with mock.patch(
- f"products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.source.{patched}"
- ) as mocked:
+ with mock.patch(f"products.warehouse_sources.sources.pipedrive.source.{patched}") as mocked:
getattr(self.source, method_name)(self.config, "https://webhooks.example/hook", self.team_id)
mocked.assert_called_once_with("acme", "token", "https://webhooks.example/hook")
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.source.pipedrive_source")
+ @mock.patch("products.warehouse_sources.sources.pipedrive.source.pipedrive_source")
def test_source_for_pipeline_passes_a_webhook_manager(self, mock_pipedrive_source: mock.MagicMock) -> None:
inputs = mock.MagicMock()
inputs.schema_name = "deals"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/tests/test_pipedrive_webhook_template.py b/products/warehouse_sources/sources/pipedrive/tests/test_pipedrive_webhook_template.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/tests/test_pipedrive_webhook_template.py
rename to products/warehouse_sources/sources/pipedrive/tests/test_pipedrive_webhook_template.py
index c3926eb9d618..60cb6b523a70 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/tests/test_pipedrive_webhook_template.py
+++ b/products/warehouse_sources/sources/pipedrive/tests/test_pipedrive_webhook_template.py
@@ -4,8 +4,8 @@ from typing import Any
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.pipedrive.webhook_template import template
+from products.warehouse_sources.sources.pipedrive.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
SOURCE_ID = "source_test_123"
AUTH_USER = "posthog"
@@ -17,7 +17,7 @@ def _basic(user: str, password: str) -> str:
return "Basic " + base64.b64encode(f"{user}:{password}".encode()).decode()
-class TestPipedriveWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestPipedriveWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/webhook_template.py b/products/warehouse_sources/sources/pipedrive/webhook_template.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/webhook_template.py
rename to products/warehouse_sources/sources/pipedrive/webhook_template.py
index 7ad7920da7e0..87fc4cdcd0fc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/pipedrive/webhook_template.py
+++ b/products/warehouse_sources/sources/pipedrive/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/sources/postmark/__init__.py b/products/warehouse_sources/sources/postmark/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/postmark.py b/products/warehouse_sources/sources/postmark/_config.py
similarity index 72%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/postmark.py
rename to products/warehouse_sources/sources/postmark/_config.py
index 80068669a929..55c5bd99c64e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/postmark.py
+++ b/products/warehouse_sources/sources/postmark/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/canonical_descriptions.py b/products/warehouse_sources/sources/postmark/canonical_descriptions.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/postmark/canonical_descriptions.py
rename to products/warehouse_sources/sources/postmark/canonical_descriptions.py
index a34882057835..bd5f57612f9a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/postmark/canonical_descriptions.py
@@ -5,9 +5,7 @@ endpoint names in `settings.py` `POSTMARK_ENDPOINTS`, which match the `ExternalD
synced Postmark table. Columns absent here fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"messages_outbound": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/postmark.py b/products/warehouse_sources/sources/postmark/postmark.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/postmark/postmark.py
rename to products/warehouse_sources/sources/postmark/postmark.py
index 5ec24330e6e5..36b542c2bb20 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/postmark.py
+++ b/products/warehouse_sources/sources/postmark/postmark.py
@@ -28,26 +28,7 @@ from asgiref.sync import async_to_sync
from dateutil import parser as dateutil_parser
from requests import Response, Session
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- OffsetPaginator,
- SinglePagePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.settings import (
+from products.warehouse_sources.sources.postmark.settings import (
POSTMARK_ENDPOINTS,
POSTMARK_MAX_PAGE_SIZE,
POSTMARK_MAX_WINDOW,
@@ -57,6 +38,21 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.s
WEBHOOK_SECRET_HEADER,
WEBHOOK_TRIGGERS,
)
+from products.warehouse_sources.sources.sdk import (
+ ExternalWebhookInfo,
+ OffsetPaginator,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SinglePagePaginator,
+ SourceResponse,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSourceManager,
+ make_tracked_session,
+ rest_api_resource,
+ validate_via_probe,
+)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
logger = logging.getLogger(__name__)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/settings.py b/products/warehouse_sources/sources/postmark/settings.py
similarity index 100%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/postmark/settings.py
rename to products/warehouse_sources/sources/postmark/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/source.py b/products/warehouse_sources/sources/postmark/source.py
similarity index 81%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/postmark/source.py
rename to products/warehouse_sources/sources/postmark/source.py
index 48b1a124061b..392bb30a5249 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/source.py
+++ b/products/warehouse_sources/sources/postmark/source.py
@@ -1,49 +1,40 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.postmark._config import PostmarkSourceConfig
+from products.warehouse_sources.sources.postmark.postmark import (
+ PostmarkResumeConfig,
+ create_webhook as create_postmark_webhook,
+ delete_webhook as delete_postmark_webhook,
+ get_external_webhook_info as get_postmark_webhook_info,
+ postmark_source,
+ validate_credentials as validate_postmark_credentials,
+)
+from products.warehouse_sources.sources.postmark.settings import ENDPOINTS, WEBHOOK_RESOURCE_MAP, WEBHOOK_SCHEMA_NAMES
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postmark import (
- PostmarkSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.postmark import (
- PostmarkResumeConfig,
- create_webhook as create_postmark_webhook,
- delete_webhook as delete_postmark_webhook,
- get_external_webhook_info as get_postmark_webhook_info,
- postmark_source,
- validate_credentials as validate_postmark_credentials,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.settings import (
- ENDPOINTS,
- WEBHOOK_RESOURCE_MAP,
- WEBHOOK_SCHEMA_NAMES,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@SourceRegistry.register
@@ -127,9 +118,7 @@ Postmark doesn't sign deliveries, so the header is what proves a delivery came f
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.postmark.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
@@ -198,9 +187,7 @@ Postmark doesn't sign deliveries, so the header is what proves a delivery came f
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.webhook_template import ( # noqa: PLC0415
- template,
- )
+ from products.warehouse_sources.sources.postmark.webhook_template import template # noqa: PLC0415
return template
diff --git a/products/warehouse_sources/sources/postmark/tests/__init__.py b/products/warehouse_sources/sources/postmark/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/tests/test_postmark.py b/products/warehouse_sources/sources/postmark/tests/test_postmark.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/postmark/tests/test_postmark.py
rename to products/warehouse_sources/sources/postmark/tests/test_postmark.py
index f8c59d141343..adf25cb0596e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/tests/test_postmark.py
+++ b/products/warehouse_sources/sources/postmark/tests/test_postmark.py
@@ -6,19 +6,8 @@ from unittest import mock
from parameterized import parameterized
from requests import Response
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- DriveResult,
- Script,
- ScriptedResponse,
- SourceDriver,
- always,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postmark import (
- PostmarkSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.postmark import (
+from products.warehouse_sources.sources.postmark._config import PostmarkSourceConfig
+from products.warehouse_sources.sources.postmark.postmark import (
POSTMARK_BASE_URL,
WEBHOOK_AUTH_ERROR,
PostmarkResumeConfig,
@@ -29,7 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.p
postmark_source,
validate_credentials,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.settings import (
+from products.warehouse_sources.sources.postmark.settings import (
ENDPOINTS,
POSTMARK_ENDPOINTS,
POSTMARK_MAX_WINDOW,
@@ -37,12 +26,21 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.s
WEBHOOK_SCHEMA_NAMES,
WEBHOOK_SECRET_HEADER,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.source import PostmarkSource
+from products.warehouse_sources.sources.postmark.source import PostmarkSource
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import (
+ DriveResult,
+ Script,
+ ScriptedResponse,
+ SourceDriver,
+ always,
+ scripted_network,
+)
# RESTClient builds its session via make_tracked_session in the rest_client module.
CLIENT_SESSION_PATCH = "products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.rest_client.make_tracked_session"
# Webhook management builds its session through a module-level helper in the postmark module.
-POSTMARK_LOGGER_PATCH = "products.warehouse_sources.backend.temporal.data_imports.sources.postmark.postmark.logger"
+POSTMARK_LOGGER_PATCH = "products.warehouse_sources.sources.postmark.postmark.logger"
def _response(body: dict[str, Any], *, status: int = 200) -> Response:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/tests/test_postmark_source.py b/products/warehouse_sources/sources/postmark/tests/test_postmark_source.py
similarity index 79%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/postmark/tests/test_postmark_source.py
rename to products/warehouse_sources/sources/postmark/tests/test_postmark_source.py
index d25281b64430..e721683abdbb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/tests/test_postmark_source.py
+++ b/products/warehouse_sources/sources/postmark/tests/test_postmark_source.py
@@ -2,11 +2,9 @@ from unittest import mock
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postmark import (
- PostmarkSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.source import PostmarkSource
+from products.warehouse_sources.sources.postmark._config import PostmarkSourceConfig
+from products.warehouse_sources.sources.postmark.source import PostmarkSource
+from products.warehouse_sources.sources.sdk import WebhookSourceManager
class TestPostmarkSource:
@@ -15,9 +13,7 @@ class TestPostmarkSource:
self.team_id = 123
self.config = PostmarkSourceConfig(server_token="test-server-token")
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.postmark.source.validate_postmark_credentials"
- )
+ @mock.patch("products.warehouse_sources.sources.postmark.source.validate_postmark_credentials")
def test_validate_credentials_success(self, mock_validate):
mock_validate.return_value = (True, 200)
@@ -33,9 +29,7 @@ class TestPostmarkSource:
("rate_limited_429", 429, "Couldn't reach Postmark"),
]
)
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.postmark.source.validate_postmark_credentials"
- )
+ @mock.patch("products.warehouse_sources.sources.postmark.source.validate_postmark_credentials")
def test_validate_credentials_failure_maps_status(self, _name, status, expected_substring, mock_validate):
mock_validate.return_value = (False, status)
@@ -45,7 +39,7 @@ class TestPostmarkSource:
assert error_message is not None
assert expected_substring in error_message
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.postmark.source.postmark_source")
+ @mock.patch("products.warehouse_sources.sources.postmark.source.postmark_source")
def test_source_for_pipeline(self, mock_postmark_source):
mock_postmark_source.return_value = mock.MagicMock()
@@ -88,9 +82,7 @@ class TestPostmarkSource:
]
)
def test_webhook_management_delegates_with_the_server_token(self, method_name, patched_name):
- with mock.patch(
- f"products.warehouse_sources.backend.temporal.data_imports.sources.postmark.source.{patched_name}"
- ) as mock_fn:
+ with mock.patch(f"products.warehouse_sources.sources.postmark.source.{patched_name}") as mock_fn:
result = getattr(self.source, method_name)(self.config, "https://ph.example/webhook", self.team_id)
mock_fn.assert_called_once_with(self.config.server_token, "https://ph.example/webhook")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/tests/test_postmark_webhook_template.py b/products/warehouse_sources/sources/postmark/tests/test_postmark_webhook_template.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/postmark/tests/test_postmark_webhook_template.py
rename to products/warehouse_sources/sources/postmark/tests/test_postmark_webhook_template.py
index f67a6583fac0..20ad91cc19c9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/tests/test_postmark_webhook_template.py
+++ b/products/warehouse_sources/sources/postmark/tests/test_postmark_webhook_template.py
@@ -2,16 +2,16 @@ import json
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.settings import WEBHOOK_SECRET_HEADER
-from products.warehouse_sources.backend.temporal.data_imports.sources.postmark.webhook_template import template
+from products.warehouse_sources.sources.postmark.settings import WEBHOOK_SECRET_HEADER
+from products.warehouse_sources.sources.postmark.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
SOURCE_ID = "source_test_123"
WEBHOOK_SECRET = "0Pm2Yb8bJ0Yl7vGQ9m1Z2xTn5cRk4dWq"
SCHEMA_MAPPING = {"Bounce": "schema_bounces"}
-class TestPostmarkWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestPostmarkWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/webhook_template.py b/products/warehouse_sources/sources/postmark/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/postmark/webhook_template.py
rename to products/warehouse_sources/sources/postmark/webhook_template.py
index fd40ffc3f6b1..c0135a600d8e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postmark/webhook_template.py
+++ b/products/warehouse_sources/sources/postmark/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
# Postmark does not sign webhook deliveries, but the Webhooks API pins static headers onto a
# webhook. `create_webhook` attaches a generated secret in the x-posthog-webhook-secret header,
diff --git a/products/warehouse_sources/sources/revenuecat/__init__.py b/products/warehouse_sources/sources/revenuecat/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/revenuecat.py b/products/warehouse_sources/sources/revenuecat/_config.py
similarity index 74%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/revenuecat.py
rename to products/warehouse_sources/sources/revenuecat/_config.py
index 9d3d2cbe0dae..1e1a60a1434c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/revenuecat.py
+++ b/products/warehouse_sources/sources/revenuecat/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/canonical_descriptions.py b/products/warehouse_sources/sources/revenuecat/canonical_descriptions.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/canonical_descriptions.py
rename to products/warehouse_sources/sources/revenuecat/canonical_descriptions.py
index 20b305d1562b..d9b057195c49 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/revenuecat/canonical_descriptions.py
@@ -6,10 +6,7 @@ Keyed by the resource names in `constants.py`, which match the `ExternalDataSche
synced RevenueCat table. Columns absent here fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.constants import (
+from products.warehouse_sources.sources.revenuecat.constants import (
APP_RESOURCE_NAME,
CUSTOMER_RESOURCE_NAME,
ENTITLEMENT_RESOURCE_NAME,
@@ -17,6 +14,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat
OFFERING_RESOURCE_NAME,
PRODUCT_RESOURCE_NAME,
)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
# Fields shared by most RevenueCat v2 objects.
_COMMON_COLUMNS = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/constants.py b/products/warehouse_sources/sources/revenuecat/constants.py
similarity index 100%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/constants.py
rename to products/warehouse_sources/sources/revenuecat/constants.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py b/products/warehouse_sources/sources/revenuecat/revenuecat.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py
rename to products/warehouse_sources/sources/revenuecat/revenuecat.py
index 541e4ad51649..f95ed05b78c9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/revenuecat.py
+++ b/products/warehouse_sources/sources/revenuecat/revenuecat.py
@@ -14,29 +14,23 @@ from urllib.parse import urlencode, urljoin
import requests
import structlog
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.revenuecat.constants import (
+ REVENUECAT_API_BASE_URL,
+ REVENUECAT_AUTO_WEBHOOK_NAME,
+)
+from products.warehouse_sources.sources.revenuecat.settings import REVENUECAT_API_ENDPOINTS
+from products.warehouse_sources.sources.sdk import (
ExternalWebhookInfo,
+ JSONResponsePaginator,
+ Resource,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
+ make_tracked_session,
rest_api_resource,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- JSONResponsePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.constants import (
- REVENUECAT_API_BASE_URL,
- REVENUECAT_AUTO_WEBHOOK_NAME,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.settings import (
- REVENUECAT_API_ENDPOINTS,
-)
LOGGER = structlog.get_logger(__name__)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/settings.py b/products/warehouse_sources/sources/revenuecat/settings.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/settings.py
rename to products/warehouse_sources/sources/revenuecat/settings.py
index 874b3ff50dec..f2b423d2229a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/settings.py
+++ b/products/warehouse_sources/sources/revenuecat/settings.py
@@ -17,7 +17,7 @@ entitlements, offerings, apps) carry ``created_at``.
from typing import NamedTuple
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.constants import (
+from products.warehouse_sources.sources.revenuecat.constants import (
APP_RESOURCE_NAME,
CUSTOMER_RESOURCE_NAME,
ENTITLEMENT_RESOURCE_NAME,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/source.py b/products/warehouse_sources/sources/revenuecat/source.py
similarity index 90%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/source.py
rename to products/warehouse_sources/sources/revenuecat/source.py
index d29e9881e27b..ca95333b18de 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/source.py
+++ b/products/warehouse_sources/sources/revenuecat/source.py
@@ -5,50 +5,43 @@ import orjson
import pyarrow as pa
from asgiref.sync import async_to_sync
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.revenuecat import revenuecat as api_client
+from products.warehouse_sources.sources.revenuecat._config import RevenueCatSourceConfig
+from products.warehouse_sources.sources.revenuecat.constants import (
+ EVENT_RESOURCE_NAME,
+ RESOURCE_TO_REVENUECAT_EVENT_TYPE,
+)
+from products.warehouse_sources.sources.revenuecat.revenuecat import RevenueCatResumeConfig
+from products.warehouse_sources.sources.revenuecat.settings import (
+ REVENUECAT_API_SCHEMA_NAMES,
+ REVENUECAT_WEBHOOK_SCHEMA_NAMES,
+)
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.revenuecat import (
- RevenueCatSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat import revenuecat as api_client
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.constants import (
- EVENT_RESOURCE_NAME,
- RESOURCE_TO_REVENUECAT_EVENT_TYPE,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat import (
- RevenueCatResumeConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.settings import (
- REVENUECAT_API_SCHEMA_NAMES,
- REVENUECAT_WEBHOOK_SCHEMA_NAMES,
-)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
REVENUECAT_API_KEYS_URL = "https://app.revenuecat.com/projects/_/api-keys"
@@ -203,9 +196,7 @@ class RevenueCatSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.webhook_template import (
- template,
- )
+ from products.warehouse_sources.sources.revenuecat.webhook_template import template
return template
@@ -292,9 +283,7 @@ class RevenueCatSource(
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.revenuecat.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/revenuecat/tests/__init__.py b/products/warehouse_sources/sources/revenuecat/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/tests/test_revenuecat.py b/products/warehouse_sources/sources/revenuecat/tests/test_revenuecat.py
similarity index 83%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/tests/test_revenuecat.py
rename to products/warehouse_sources/sources/revenuecat/tests/test_revenuecat.py
index 5121d2778082..bda2726044da 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/tests/test_revenuecat.py
+++ b/products/warehouse_sources/sources/revenuecat/tests/test_revenuecat.py
@@ -5,22 +5,12 @@ from unittest.mock import MagicMock, patch
import requests
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- DriveResult,
- ScriptedResponse,
- SourceDriver,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.revenuecat import (
- RevenueCatSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat import revenuecat as api_client
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.constants import (
- REVENUECAT_API_BASE_URL,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat import (
- RevenueCatResumeConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.source import RevenueCatSource
+from products.warehouse_sources.sources.revenuecat import revenuecat as api_client
+from products.warehouse_sources.sources.revenuecat._config import RevenueCatSourceConfig
+from products.warehouse_sources.sources.revenuecat.constants import REVENUECAT_API_BASE_URL
+from products.warehouse_sources.sources.revenuecat.revenuecat import RevenueCatResumeConfig
+from products.warehouse_sources.sources.revenuecat.source import RevenueCatSource
+from products.warehouse_sources.sources.sdk.testing import DriveResult, ScriptedResponse, SourceDriver
def _api_response(items: list | None = None, *, status: int = 200, next_page: str | None = None) -> ScriptedResponse:
@@ -54,7 +44,7 @@ def _http_error_response(status_code: int) -> MagicMock:
class TestValidateCredentials:
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_returns_true_when_projects_list_succeeds(self, mock_session):
mock_session.return_value.get.return_value = _ok_json_response({"items": []})
@@ -65,7 +55,7 @@ class TestValidateCredentials:
called_url = mock_session.return_value.get.call_args.args[0]
assert called_url == f"{REVENUECAT_API_BASE_URL}/projects"
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_follows_pagination_when_project_is_on_a_later_page(self, mock_session):
mock_session.return_value.get.side_effect = [
_ok_json_response({"items": [{"id": "proj_a"}], "next_page": "/v2/projects?starting_after=proj_a"}),
@@ -84,7 +74,7 @@ class TestValidateCredentials:
(429, "rate-limited"),
]
)
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_returns_false_on_http_error(self, status_code, expected_substring, mock_session):
mock_session.return_value.get.return_value = _http_error_response(status_code)
@@ -103,7 +93,7 @@ class TestValidateCredentials:
("project_missing", [404], False, "could not find"),
]
)
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_checks_project_endpoints_when_listing_projects_is_denied(
self, _name, endpoint_statuses, expected_success, expected_substring, mock_session
):
@@ -135,7 +125,7 @@ class TestValidateCredentials:
assert success is False
assert error == api_client.UNSUPPORTED_KEY_CHARACTER_MESSAGE
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_fails_open_on_invalid_json_on_projects_list(self, mock_session):
# If `GET /projects` returns a 200 with an unparseable body, the key is
# good but we can't run the membership check — accept rather than block.
@@ -229,10 +219,8 @@ class TestMsToSeconds:
class TestCreateWebhook:
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._find_webhook_integration"
- )
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._find_webhook_integration")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_treats_existing_webhook_as_success_with_pending_authorization(self, mock_session, mock_find):
mock_find.return_value = {"id": "wh_existing", "url": "https://example.com/h"}
@@ -242,10 +230,8 @@ class TestCreateWebhook:
assert result.pending_inputs == ["authorization_header"]
mock_session.return_value.post.assert_not_called()
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._find_webhook_integration"
- )
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._find_webhook_integration")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_updates_existing_webhook_in_place_when_authorization_header_supplied(self, mock_session, mock_find):
# Binding the header must not delete + recreate the integration —
# RevenueCat supports in-place updates via a POST to the integration's
@@ -273,10 +259,8 @@ class TestCreateWebhook:
"authorization_header": "Bearer my-secret",
}
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._find_webhook_integration"
- )
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._find_webhook_integration")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_reports_pending_authorization_header_when_not_supplied(self, mock_session, mock_find):
mock_find.return_value = None
mock_session.return_value.post.return_value = _ok_json_response({"id": "wh_1"})
@@ -288,10 +272,8 @@ class TestCreateWebhook:
body = mock_session.return_value.post.call_args.kwargs["json"]
assert "authorization_header" not in body
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._find_webhook_integration"
- )
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._find_webhook_integration")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_translates_403_into_permission_error(self, mock_session, mock_find):
mock_find.return_value = None
mock_session.return_value.post.return_value = _http_error_response(403)
@@ -311,10 +293,8 @@ class TestCreateWebhook:
class TestDeleteWebhook:
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._list_webhook_integrations"
- )
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._list_webhook_integrations")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_deletes_matching_webhook(self, mock_session, mock_list):
mock_list.return_value = [{"id": "wh_1", "url": "https://example.com/h"}]
mock_session.return_value.delete.return_value = _ok_json_response()
@@ -325,10 +305,8 @@ class TestDeleteWebhook:
called_url = mock_session.return_value.delete.call_args.args[0]
assert called_url == f"{REVENUECAT_API_BASE_URL}/projects/proj_test/integrations/webhooks/wh_1"
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._list_webhook_integrations"
- )
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._list_webhook_integrations")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_returns_success_when_webhook_not_found(self, mock_session, mock_list):
# Idempotent delete — missing webhook is treated as already gone, not
# as a failure to surface to users.
@@ -339,10 +317,8 @@ class TestDeleteWebhook:
assert result.success is True
mock_session.return_value.delete.assert_not_called()
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._list_webhook_integrations"
- )
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._list_webhook_integrations")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_404_on_delete_treated_as_success(self, mock_session, mock_list):
mock_list.return_value = [{"id": "wh_1", "url": "https://example.com/h"}]
mock_session.return_value.delete.return_value = _ok_json_response(status_code=404)
@@ -353,9 +329,7 @@ class TestDeleteWebhook:
class TestGetExternalWebhookInfo:
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._list_webhook_integrations"
- )
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._list_webhook_integrations")
def test_returns_info_for_matching_webhook(self, mock_list):
# The integration object names its subscription field `event_types`
# (lowercase values) — reading the webhook-delivery-payload spelling
@@ -380,9 +354,7 @@ class TestGetExternalWebhookInfo:
assert info.status == "enabled"
assert info.description == "PostHog data warehouse"
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._list_webhook_integrations"
- )
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._list_webhook_integrations")
def test_returns_not_found_when_no_match(self, mock_list):
mock_list.return_value = [{"id": "wh_1", "url": "https://other.example.com"}]
@@ -432,7 +404,7 @@ class TestAccessibleProjectIds:
class TestValidateCredentialsProjectSuggestions:
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_miss_lists_single_accessible_project(self, mock_session):
# The key works (it can list projects) but the entered id doesn't exist.
# The error should name the one project the key can actually reach.
@@ -449,7 +421,7 @@ class TestValidateCredentialsProjectSuggestions:
# Project names are deliberately never surfaced (they land in analytics).
assert "My App" not in error
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_miss_lists_multiple_accessible_projects(self, mock_session):
mock_session.return_value.get.return_value = _ok_json_response({"items": [{"id": "proj_a"}, {"id": "proj_b"}]})
@@ -459,7 +431,7 @@ class TestValidateCredentialsProjectSuggestions:
assert error is not None
assert "proj_a" in error and "proj_b" in error
- @patch("products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.revenuecat._session")
+ @patch("products.warehouse_sources.sources.revenuecat.revenuecat._session")
def test_miss_with_no_accessible_projects_falls_back_to_generic_message(self, mock_session):
mock_session.return_value.get.return_value = _ok_json_response({"items": []})
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/tests/test_revenuecat_source.py b/products/warehouse_sources/sources/revenuecat/tests/test_revenuecat_source.py
similarity index 85%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/tests/test_revenuecat_source.py
rename to products/warehouse_sources/sources/revenuecat/tests/test_revenuecat_source.py
index 5e8d7ddbb92c..12ef1a64b87d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/tests/test_revenuecat_source.py
+++ b/products/warehouse_sources/sources/revenuecat/tests/test_revenuecat_source.py
@@ -4,24 +4,16 @@ from unittest.mock import MagicMock, patch
import pyarrow as pa
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import WebhookCreationResult
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.revenuecat import (
- RevenueCatSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.constants import (
+from products.warehouse_sources.sources.revenuecat._config import RevenueCatSourceConfig
+from products.warehouse_sources.sources.revenuecat.constants import (
CUSTOMER_RESOURCE_NAME,
EVENT_RESOURCE_NAME,
RESOURCE_TO_REVENUECAT_EVENT_TYPE,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.settings import (
- REVENUECAT_API_ENDPOINTS,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.source import (
- RevenueCatSource,
- _webhook_table_transformer,
-)
+from products.warehouse_sources.sources.revenuecat.settings import REVENUECAT_API_ENDPOINTS
+from products.warehouse_sources.sources.revenuecat.source import RevenueCatSource, _webhook_table_transformer
+from products.warehouse_sources.sources.sdk import SourceResponse, WebhookCreationResult
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
def _config(api_key: str = "sk_test", project_id: str = "proj_test") -> RevenueCatSourceConfig:
@@ -41,9 +33,7 @@ class TestRevenueCatSourceWebhookResourceMap:
class TestRevenueCatSourceCreateWebhook:
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.source.api_client.create_webhook"
- )
+ @patch("products.warehouse_sources.sources.revenuecat.source.api_client.create_webhook")
def test_delegates_to_api_client_without_authorization_header(self, mock_create):
mock_create.return_value = WebhookCreationResult(success=True, pending_inputs=["authorization_header"])
source = RevenueCatSource()
@@ -60,12 +50,8 @@ class TestRevenueCatSourceCreateWebhook:
class TestRevenueCatSourceWebhookInputsUpdated:
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.source.api_client.delete_webhook"
- )
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.source.api_client.create_webhook"
- )
+ @patch("products.warehouse_sources.sources.revenuecat.source.api_client.delete_webhook")
+ @patch("products.warehouse_sources.sources.revenuecat.source.api_client.create_webhook")
def test_binds_authorization_header_without_deleting_integration(self, mock_create, mock_delete):
# Binding the header goes through `create_webhook`, which updates the
# existing integration in place (or creates it fresh if
@@ -88,9 +74,7 @@ class TestRevenueCatSourceWebhookInputsUpdated:
kwargs = mock_create.call_args.kwargs
assert kwargs["authorization_header_value"] == "Bearer my-secret"
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.source.api_client.create_webhook"
- )
+ @patch("products.warehouse_sources.sources.revenuecat.source.api_client.create_webhook")
def test_skips_api_call_when_authorization_header_missing(self, mock_create):
source = RevenueCatSource()
@@ -100,9 +84,7 @@ class TestRevenueCatSourceWebhookInputsUpdated:
assert error is None
mock_create.assert_not_called()
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.source.api_client.create_webhook"
- )
+ @patch("products.warehouse_sources.sources.revenuecat.source.api_client.create_webhook")
def test_propagates_bind_failure(self, mock_create):
mock_create.return_value = WebhookCreationResult(success=False, error="boom")
source = RevenueCatSource()
@@ -116,9 +98,7 @@ class TestRevenueCatSourceWebhookInputsUpdated:
class TestRevenueCatSourcePipelineDispatch:
- @patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.revenuecat.source.api_client.revenuecat_api_source"
- )
+ @patch("products.warehouse_sources.sources.revenuecat.source.api_client.revenuecat_api_source")
def test_api_schema_delegates_to_transport_with_config_and_inputs(self, mock_api_source):
# Resume scoping and pagination now live inside `revenuecat_api_source`; the source just
# wires the config secret/project and the run's team/job/schema through to it.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/webhook_template.py b/products/warehouse_sources/sources/revenuecat/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/webhook_template.py
rename to products/warehouse_sources/sources/revenuecat/webhook_template.py
index 9fbf7564578f..09d4eb38e4e1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/revenuecat/webhook_template.py
+++ b/products/warehouse_sources/sources/revenuecat/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/sources/sdk/__init__.py b/products/warehouse_sources/sources/sdk/__init__.py
index c606d1b7c109..b1edfaa8f28b 100644
--- a/products/warehouse_sources/sources/sdk/__init__.py
+++ b/products/warehouse_sources/sources/sdk/__init__.py
@@ -11,6 +11,7 @@ Django settings. That is not true yet: several origin modules import Django mode
from posthog.cloud_utils import is_cloud
from posthog.dataclasses import frozen
+from posthog.egress.slack.transport import slack_request
from posthog.exceptions_capture import capture_exception
from posthog.integrations import IntegrationKind, IntegrationManager, IntegrationNotFound, IntegrationRecord
from posthog.models.integration import (
@@ -25,6 +26,7 @@ from posthog.security.url_validation import is_url_allowed
from posthog.temporal.common.errors import NonReportableError
from posthog.temporal.common.shutdown import WorkerShuttingDownError
+from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
from products.warehouse_sources.backend.facade.codebase_detection import NOT_DETECTABLE, detect
from products.warehouse_sources.backend.facade.source_config import (
DataWarehouseSourceCategory,
@@ -59,10 +61,15 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common imp
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
UNVERSIONED_API_VERSION,
+ ExternalWebhookInfo,
FieldType,
ResumableSource,
SimpleSource,
VersionDeprecation,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSource,
+ WebhookSyncResult,
)
from products.warehouse_sources.backend.temporal.data_imports.sources.common.boundary_checkpoint import (
BoundaryCheckpoint,
@@ -131,11 +138,13 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.mix
DATABASE_HOST_NOT_ALLOWED_ERROR,
HostNotAllowedError,
OAuthMixin,
+ TemporaryHostResolutionError,
ValidateDatabaseHostMixin,
_is_host_safe,
log_connection_open,
resolve_safe_host,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.oauth_pointers import PointerPath
from products.warehouse_sources.backend.temporal.data_imports.sources.common.primary_keys import resolve_merge_keys
from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import (
SourceKey,
@@ -279,6 +288,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
SourceResponse,
TDataType,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
from products.warehouse_sources.backend.temporal.data_imports.sources.salesforce.auth import (
salesforce_refresh_access_token,
)
@@ -316,6 +326,7 @@ __all__ = [
"ExternalDataSchema",
"ExternalDataSchemaSyncType",
"ExternalDataSourceType",
+ "ExternalWebhookInfo",
"FILE_MODIFIED_AT_COLUMN",
"FILE_PATH_COLUMN",
"FORMAT_ERROR",
@@ -324,6 +335,7 @@ __all__ = [
"FileDelimiterError",
"HTTPMethodBasic",
"HeaderLinkPaginator",
+ "HogFunctionTemplateDC",
"HostNotAllowedError",
"HttpBasicAuth",
"HttpBasicAuthConfig",
@@ -357,6 +369,7 @@ __all__ = [
"OffsetPaginator",
"PageNumberPaginator",
"PageNumberPaginatorConfig",
+ "PointerPath",
"PaginatorConfig",
"ParamLocation",
"ParamStyle",
@@ -405,6 +418,7 @@ __all__ = [
"TJsonPath",
"Table",
"TableStats",
+ "TemporaryHostResolutionError",
"TrackedHTTPAdapter",
"UNKNOWN_RESOURCE_PREFIX",
"UNVERSIONED_API_VERSION",
@@ -412,6 +426,11 @@ __all__ = [
"ValidateDatabaseHostMixin",
"ValidatedRowFilter",
"VersionDeprecation",
+ "WebhookCreationResult",
+ "WebhookDeletionResult",
+ "WebhookSource",
+ "WebhookSourceManager",
+ "WebhookSyncResult",
"WorkerShuttingDownError",
"_NoRedirectSession",
"_inject_param",
@@ -472,6 +491,7 @@ __all__ = [
"schema_for_resource",
"scrub_url",
"set_json_param",
+ "slack_request",
"source_helpers",
"source_key",
"stop_at_request_deadline",
diff --git a/products/warehouse_sources/sources/sdk/testing.py b/products/warehouse_sources/sources/sdk/testing.py
index 74b7b13b6494..c0f49e40ee6a 100644
--- a/products/warehouse_sources/sources/sdk/testing.py
+++ b/products/warehouse_sources/sources/sdk/testing.py
@@ -10,6 +10,8 @@ from posthog.hogql.parser import parse_select
from posthog.models.integration.model import Integration
from posthog.test.anti_ssrf import mock_anti_ssrf_dns_lookup
+from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateSimpleTest
+from products.cdp.backend.validation import compile_hog
from products.warehouse_sources.backend.temporal.data_imports.sources.common import boundary_checkpoint, resumable
from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches
from products.warehouse_sources.backend.temporal.data_imports.sources.common.file_parsing import FileFormatError
@@ -41,7 +43,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
route,
)
+from common.hogvm.python.execute import execute_bytecode
+
__all__ = [
+ "BaseHogFunctionTemplateSimpleTest",
"ColumnTypeCategory",
"DEFAULT_RETRY_ATTEMPTS",
"DriveResult",
@@ -60,7 +65,9 @@ __all__ = [
"boundary_checkpoint",
"build_default_schemas",
"build_default_sync_settings",
+ "compile_hog",
"error_message_matches",
+ "execute_bytecode",
"mock_anti_ssrf_dns_lookup",
"parse_select",
"rest_client",
diff --git a/products/warehouse_sources/sources/slack/__init__.py b/products/warehouse_sources/sources/slack/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/slack.py b/products/warehouse_sources/sources/slack/_config.py
similarity index 89%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/slack.py
rename to products/warehouse_sources/sources/slack/_config.py
index dbd722cb6046..78afc66c48c0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/slack.py
+++ b/products/warehouse_sources/sources/slack/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/canonical_descriptions.py b/products/warehouse_sources/sources/slack/canonical_descriptions.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/slack/canonical_descriptions.py
rename to products/warehouse_sources/sources/slack/canonical_descriptions.py
index 1ee5851296d6..91f01f6df9f2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/slack/canonical_descriptions.py
@@ -7,9 +7,7 @@ tables are discovered dynamically (keyed by channel ID) and are not documented h
here fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"$channels": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/settings.py b/products/warehouse_sources/sources/slack/settings.py
similarity index 78%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/slack/settings.py
rename to products/warehouse_sources/sources/slack/settings.py
index fb6e587852a4..d785df5ef724 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/settings.py
+++ b/products/warehouse_sources/sources/slack/settings.py
@@ -1,11 +1,7 @@
from dataclasses import dataclass, field
from typing import Optional
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import (
- PartitionFormat,
- PartitionMode,
-)
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.sources.sdk import IncrementalField, PartitionFormat, PartitionMode
@dataclass
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/slack.py b/products/warehouse_sources/sources/slack/slack.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/slack/slack.py
rename to products/warehouse_sources/sources/slack/slack.py
index ac5b994534ab..251678f018a7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/slack.py
+++ b/products/warehouse_sources/sources/slack/slack.py
@@ -16,24 +16,21 @@ from requests import Request, Response
from tenacity import RetryCallState, retry, retry_if_exception_type, stop_after_attempt, wait_exponential_jitter
from tenacity.stop import stop_base
-from posthog.egress.slack.transport import slack_request
-
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
+from products.warehouse_sources.sources.sdk import (
+ BasePaginator,
+ BearerTokenAuth,
+ EndpointResource,
RESTAPIConfig,
+ ResumableSourceManager,
+ SortMode,
+ SourceResponse,
+ WebhookSourceManager,
+ make_tracked_session,
rest_api_resource,
+ slack_request,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import BearerTokenAuth
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SortMode, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.slack.settings import (
- ENDPOINTS,
- messages_endpoint_config,
-)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.slack.settings import ENDPOINTS, messages_endpoint_config
logger = structlog.get_logger(__name__)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/source.py b/products/warehouse_sources/sources/slack/source.py
similarity index 91%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/slack/source.py
rename to products/warehouse_sources/sources/slack/source.py
index 7466fdec8ff7..2d022278610e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/source.py
+++ b/products/warehouse_sources/sources/slack/source.py
@@ -2,45 +2,40 @@ from collections.abc import Mapping
from dataclasses import field
from typing import TYPE_CHECKING, Any, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
+from products.warehouse_sources.sources.sdk import detect
if TYPE_CHECKING:
- from posthog.integrations import IntegrationRecord
-
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC, IntegrationRecord
from temporalio import activity
-from posthog.dataclasses import frozen
-
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ FieldType,
+ OAuthMixin,
+ PointerPath,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
SourceFieldSwitchGroupConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
+ frozen,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import OAuthMixin
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.oauth_pointers import PointerPath
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.slack import SlackSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.slack.settings import ENDPOINTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack import (
+from products.warehouse_sources.sources.slack._config import SlackSourceConfig
+from products.warehouse_sources.sources.slack.settings import ENDPOINTS
+from products.warehouse_sources.sources.slack.slack import (
SlackResumeConfig,
auth_test_user_id,
get_channels,
@@ -49,7 +44,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.slack.slac
slack_source,
validate_credentials as validate_slack_credentials,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
# Joining is one rate-limited call per channel, so a large workspace can't finish in one pass. The setup
# wizard and a manual refresh discover schemas inside a web request, which the gateway ends at 120
@@ -150,7 +144,7 @@ class SlackSource(ResumableSource[SlackSourceConfig, SlackResumeConfig], Webhook
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.slack.webhook_template import template
+ from products.warehouse_sources.sources.slack.webhook_template import template
return template
@@ -360,9 +354,7 @@ Prefer a manifest? Paste this when creating the app — it wires the request URL
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.slack.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.slack.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/test_slack.py b/products/warehouse_sources/sources/slack/test_slack.py
similarity index 85%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/slack/test_slack.py
rename to products/warehouse_sources/sources/slack/test_slack.py
index 9e52907812ca..c705339ebf06 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/test_slack.py
+++ b/products/warehouse_sources/sources/slack/test_slack.py
@@ -7,18 +7,17 @@ from django.core.cache import cache as django_cache
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+from products.warehouse_sources.sources.sdk import ResumableSourceManager, WebhookSourceManager
+from products.warehouse_sources.sources.sdk.testing import (
RecordedRequest,
ScriptedResponse,
SourceDriver,
always,
scripted_network,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.slack import SlackSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.slack.settings import ENDPOINTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack import (
+from products.warehouse_sources.sources.slack._config import SlackSourceConfig
+from products.warehouse_sources.sources.slack.settings import ENDPOINTS
+from products.warehouse_sources.sources.slack.slack import (
SlackResumeConfig,
_channel_messages_generator,
_fetch_all_channels,
@@ -29,7 +28,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.slack.slac
manual_cache_id,
slack_source,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.slack.source import SlackSource
+from products.warehouse_sources.sources.slack.source import SlackSource
def _make_response(payload: dict[str, Any]) -> MagicMock:
@@ -129,7 +128,7 @@ class TestChannelMessagesGeneratorResumable:
responses = [_make_response(p) for p in pages]
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._slack_get",
+ "products.warehouse_sources.sources.slack.slack._slack_get",
side_effect=responses,
):
items = list(_channel_messages_generator("token", "C123", resume_mgr, oldest_ts=oldest_ts))
@@ -149,7 +148,7 @@ class TestChannelMessagesGeneratorResumable:
responses = [_make_response(p) for p in pages]
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._slack_get",
+ "products.warehouse_sources.sources.slack.slack._slack_get",
side_effect=responses,
) as mock_get:
items = list(_channel_messages_generator("token", "C123", resume_mgr, oldest_ts=None))
@@ -171,7 +170,7 @@ class TestChannelMessagesGeneratorResumable:
responses = [_make_response(p) for p in pages]
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._slack_get",
+ "products.warehouse_sources.sources.slack.slack._slack_get",
side_effect=responses,
) as mock_get:
list(_channel_messages_generator("token", "C123", resume_mgr, oldest_ts="original_oldest"))
@@ -225,7 +224,7 @@ class TestFetchChannelsByType:
) -> None:
responses = [_make_response(_make_channel_page([{"id": "X1", "name": "x"}]))]
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._slack_get",
+ "products.warehouse_sources.sources.slack.slack._slack_get",
side_effect=responses,
) as mock_get:
channels = _fetch_channels_by_type("token", channel_type, authed_user=authed_user)
@@ -246,7 +245,7 @@ class TestFetchChannelsByType:
]
responses = [_make_response(p) for p in pages]
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._slack_get",
+ "products.warehouse_sources.sources.slack.slack._slack_get",
side_effect=responses,
) as mock_get:
channels = _fetch_channels_by_type("token", "public_channel")
@@ -264,7 +263,7 @@ class TestFetchAllChannels:
]
responses = [_make_response(p) for p in pages]
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._slack_get",
+ "products.warehouse_sources.sources.slack.slack._slack_get",
side_effect=responses,
) as mock_get:
channels = _fetch_all_channels("token", authed_user="U_INSTALLER")
@@ -288,7 +287,7 @@ class TestFetchAllChannelsCached:
def test_second_call_uses_cache(self) -> None:
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._fetch_all_channels",
+ "products.warehouse_sources.sources.slack.slack._fetch_all_channels",
return_value=[{"id": "C1", "name": "general"}],
) as mock_fetch:
first = _fetch_all_channels_cached(cache_id="42", access_token="token", authed_user="U_INSTALLER")
@@ -299,7 +298,7 @@ class TestFetchAllChannelsCached:
def test_different_integrations_get_independent_cache_entries(self) -> None:
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._fetch_all_channels",
+ "products.warehouse_sources.sources.slack.slack._fetch_all_channels",
side_effect=[[{"id": "A1", "name": "a"}], [{"id": "B1", "name": "b"}]],
) as mock_fetch:
a = _fetch_all_channels_cached(cache_id="1", access_token="token", authed_user="U1")
@@ -311,7 +310,7 @@ class TestFetchAllChannelsCached:
def test_force_refresh_refetches_and_overwrites_cache(self) -> None:
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._fetch_all_channels",
+ "products.warehouse_sources.sources.slack.slack._fetch_all_channels",
side_effect=[[{"id": "C1", "name": "general"}], [{"id": "C2", "name": "renamed"}]],
) as mock_fetch:
first = _fetch_all_channels_cached(cache_id="42", access_token="token", authed_user="U_INSTALLER")
@@ -330,13 +329,13 @@ class TestFetchAllChannelsCached:
# If the upstream fetch raises, the previous cached value must remain so
# concurrent readers continue to see stale-but-valid data.
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._fetch_all_channels",
+ "products.warehouse_sources.sources.slack.slack._fetch_all_channels",
return_value=[{"id": "C1", "name": "general"}],
):
_fetch_all_channels_cached(cache_id="42", access_token="token", authed_user="U_INSTALLER")
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._fetch_all_channels",
+ "products.warehouse_sources.sources.slack.slack._fetch_all_channels",
side_effect=RuntimeError("rate limited"),
):
try:
@@ -347,7 +346,7 @@ class TestFetchAllChannelsCached:
pass
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._fetch_all_channels",
+ "products.warehouse_sources.sources.slack.slack._fetch_all_channels",
return_value=[{"id": "WRONG", "name": "should_not_be_called"}],
) as mock_fetch:
after = _fetch_all_channels_cached(cache_id="42", access_token="token", authed_user="U_INSTALLER")
@@ -376,7 +375,7 @@ class TestSlackSourceGetSchemasForceRefresh:
return config, integration
def test_force_refresh_bypasses_channels_cache(self) -> None:
- from products.warehouse_sources.backend.temporal.data_imports.sources.slack.source import SlackSource
+ from products.warehouse_sources.sources.slack.source import SlackSource
config, integration = self._build_mocks()
source = SlackSource()
@@ -384,7 +383,7 @@ class TestSlackSourceGetSchemasForceRefresh:
with (
patch.object(source, "get_oauth_integration", return_value=integration),
patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._fetch_all_channels",
+ "products.warehouse_sources.sources.slack.slack._fetch_all_channels",
side_effect=[[{"id": "C1", "name": "general"}], [{"id": "C2", "name": "renamed"}]],
) as mock_fetch,
):
@@ -404,7 +403,7 @@ class TestSlackSourceGetSchemasForceRefresh:
assert channel_names(forced) == {"C2"}
def test_channel_schemas_are_webhook_only(self) -> None:
- from products.warehouse_sources.backend.temporal.data_imports.sources.slack.source import SlackSource
+ from products.warehouse_sources.sources.slack.source import SlackSource
config, integration = self._build_mocks()
source = SlackSource()
@@ -412,7 +411,7 @@ class TestSlackSourceGetSchemasForceRefresh:
with (
patch.object(source, "get_oauth_integration", return_value=integration),
patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._fetch_all_channels",
+ "products.warehouse_sources.sources.slack.slack._fetch_all_channels",
return_value=[{"id": "C1", "name": "general"}],
),
):
@@ -505,9 +504,7 @@ class TestSlackSourceChannelMessagesWebhookOnly:
manager.webhook_enabled = AsyncMock(return_value=webhook_on)
manager.get_items.return_value = ["webhook-item"]
- with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._channel_messages_generator"
- ) as mock_backfill:
+ with patch("products.warehouse_sources.sources.slack.slack._channel_messages_generator") as mock_backfill:
response = self._build_channel_source(manager)
items = list(response.items())
@@ -531,13 +528,11 @@ class TestSlackSourceChannelMessagesWebhookOnly:
class TestResolveAccessToken:
- _SOURCE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.slack.source"
+ _SOURCE_MODULE = "products.warehouse_sources.sources.slack.source"
def test_token_uses_pasted_token_without_an_integration(self) -> None:
- from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.slack import (
- SlackSourceConfig,
- )
- from products.warehouse_sources.backend.temporal.data_imports.sources.slack.source import SlackSource
+ from products.warehouse_sources.sources.slack._config import SlackSourceConfig
+ from products.warehouse_sources.sources.slack.source import SlackSource
config = SlackSourceConfig.from_dict({"slack_access_token": "xoxb-abc"})
source = SlackSource()
@@ -557,13 +552,8 @@ class TestResolveAccessToken:
mock_auth_test.assert_called_once_with("xoxb-abc")
def test_legacy_source_resolves_via_linked_integration(self) -> None:
- from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.slack import (
- SlackSourceConfig,
- )
- from products.warehouse_sources.backend.temporal.data_imports.sources.slack.source import (
- SlackAccess,
- SlackSource,
- )
+ from products.warehouse_sources.sources.slack._config import SlackSourceConfig
+ from products.warehouse_sources.sources.slack.source import SlackAccess, SlackSource
config = SlackSourceConfig.from_dict({"slack_integration_id": 42})
source = SlackSource()
@@ -582,10 +572,8 @@ class TestResolveAccessToken:
def test_token_takes_precedence_over_a_legacy_integration(self) -> None:
# Converting a legacy source in place: once a bot token is pasted, the stale integration id
# is ignored and no Integration lookup happens.
- from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.slack import (
- SlackSourceConfig,
- )
- from products.warehouse_sources.backend.temporal.data_imports.sources.slack.source import SlackSource
+ from products.warehouse_sources.sources.slack._config import SlackSourceConfig
+ from products.warehouse_sources.sources.slack.source import SlackSource
config = SlackSourceConfig.from_dict({"slack_access_token": "xoxb-new", "slack_integration_id": 42})
source = SlackSource()
@@ -600,10 +588,8 @@ class TestResolveAccessToken:
mock_get_integration.assert_not_called()
def test_no_credentials_raises(self) -> None:
- from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.slack import (
- SlackSourceConfig,
- )
- from products.warehouse_sources.backend.temporal.data_imports.sources.slack.source import SlackSource
+ from products.warehouse_sources.sources.slack._config import SlackSourceConfig
+ from products.warehouse_sources.sources.slack.source import SlackSource
config = SlackSourceConfig.from_dict({})
source = SlackSource()
@@ -625,7 +611,7 @@ class TestAuthTestUserId:
)
def test_parses_response(self, _name: str, payload: dict[str, Any], expected: str | None) -> None:
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._slack_get",
+ "products.warehouse_sources.sources.slack.slack._slack_get",
return_value=_make_response(payload),
):
assert auth_test_user_id("token") == expected
@@ -634,14 +620,14 @@ class TestAuthTestUserId:
# A failed auth.test must not blow up discovery — it falls back to unscoped private-channel
# listing rather than crashing the sync.
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._slack_get",
+ "products.warehouse_sources.sources.slack.slack._slack_get",
side_effect=RuntimeError("boom"),
):
assert auth_test_user_id("token") is None
class TestJoinPublicChannels:
- _POST = "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._slack_post"
+ _POST = "products.warehouse_sources.sources.slack.slack._slack_post"
def test_only_public_non_member_channels_are_joined(self) -> None:
channels: list[dict[str, Any]] = [
@@ -691,7 +677,7 @@ class TestJoinPublicChannels:
class TestGetSchemasAutoJoin:
- _SRC = "products.warehouse_sources.backend.temporal.data_imports.sources.slack.source"
+ _SRC = "products.warehouse_sources.sources.slack.source"
def setup_method(self) -> None:
django_cache.clear()
@@ -709,7 +695,7 @@ class TestGetSchemasAutoJoin:
def test_join_is_gated_on_toggle_and_byo_token(
self, _name: str, enabled: bool, token: str | None, expect_join: bool
) -> None:
- from products.warehouse_sources.backend.temporal.data_imports.sources.slack.source import SlackSource
+ from products.warehouse_sources.sources.slack.source import SlackSource
config = MagicMock()
config.slack_access_token = token
@@ -735,10 +721,8 @@ class TestGetSchemasAutoJoin:
def test_byo_source_joins_by_default(self) -> None:
# The toggle defaults on: a bring-your-own source that doesn't set it explicitly still
# auto-joins. (Legacy OAuth sources are held off by the token gate — covered above.)
- from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.slack import (
- SlackSourceConfig,
- )
- from products.warehouse_sources.backend.temporal.data_imports.sources.slack.source import SlackSource
+ from products.warehouse_sources.sources.slack._config import SlackSourceConfig
+ from products.warehouse_sources.sources.slack.source import SlackSource
config = SlackSourceConfig.from_dict({"slack_access_token": "xoxb-token"})
assert config.join_public_channels is not None
@@ -756,10 +740,8 @@ class TestGetSchemasAutoJoin:
assert mock_join.call_args.kwargs["time_budget_seconds"] is not None
def test_toggle_can_be_explicitly_disabled(self) -> None:
- from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.slack import (
- SlackSourceConfig,
- )
- from products.warehouse_sources.backend.temporal.data_imports.sources.slack.source import SlackSource
+ from products.warehouse_sources.sources.slack._config import SlackSourceConfig
+ from products.warehouse_sources.sources.slack.source import SlackSource
config = SlackSourceConfig.from_dict(
{"slack_access_token": "xoxb-token", "join_public_channels": {"enabled": False}}
diff --git a/products/warehouse_sources/sources/slack/tests/__init__.py b/products/warehouse_sources/sources/slack/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/tests/test_slack_source.py b/products/warehouse_sources/sources/slack/tests/test_slack_source.py
similarity index 86%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/slack/tests/test_slack_source.py
rename to products/warehouse_sources/sources/slack/tests/test_slack_source.py
index f3d2092b149f..2b49114a6980 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/tests/test_slack_source.py
+++ b/products/warehouse_sources/sources/slack/tests/test_slack_source.py
@@ -3,16 +3,12 @@ from unittest.mock import MagicMock
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import WebhookSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack import (
- validate_credentials as validate_slack_credentials,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.slack.source import SlackAccess, SlackSource
-
-SOURCE_VALIDATE_PATCH = (
- "products.warehouse_sources.backend.temporal.data_imports.sources.slack.source.validate_slack_credentials"
-)
-SLACK_GET_PATCH = "products.warehouse_sources.backend.temporal.data_imports.sources.slack.slack._slack_get"
+from products.warehouse_sources.sources.sdk import WebhookSource
+from products.warehouse_sources.sources.slack.slack import validate_credentials as validate_slack_credentials
+from products.warehouse_sources.sources.slack.source import SlackAccess, SlackSource
+
+SOURCE_VALIDATE_PATCH = "products.warehouse_sources.sources.slack.source.validate_slack_credentials"
+SLACK_GET_PATCH = "products.warehouse_sources.sources.slack.slack._slack_get"
class TestSlackValidateCredentialsProbe:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/tests/test_slack_webhook_template.py b/products/warehouse_sources/sources/slack/tests/test_slack_webhook_template.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/slack/tests/test_slack_webhook_template.py
rename to products/warehouse_sources/sources/slack/tests/test_slack_webhook_template.py
index 3fb5b0c89d69..7d0e64d37797 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/tests/test_slack_webhook_template.py
+++ b/products/warehouse_sources/sources/slack/tests/test_slack_webhook_template.py
@@ -5,11 +5,11 @@ import hashlib
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.slack.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
+from products.warehouse_sources.sources.slack.webhook_template import template
-class TestSlackWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestSlackWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/webhook_template.py b/products/warehouse_sources/sources/slack/webhook_template.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/slack/webhook_template.py
rename to products/warehouse_sources/sources/slack/webhook_template.py
index d5ff8196f565..58cced08dd68 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/slack/webhook_template.py
+++ b/products/warehouse_sources/sources/slack/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/sources/sparkpost/__init__.py b/products/warehouse_sources/sources/sparkpost/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/sparkpost.py b/products/warehouse_sources/sources/sparkpost/_config.py
similarity index 77%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/sparkpost.py
rename to products/warehouse_sources/sources/sparkpost/_config.py
index 16dd198367d8..f3d20ca0c7d9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/sparkpost.py
+++ b/products/warehouse_sources/sources/sparkpost/_config.py
@@ -3,7 +3,7 @@
from typing import Literal
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/api_inventory.md b/products/warehouse_sources/sources/sparkpost/api_inventory.md
similarity index 100%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/api_inventory.md
rename to products/warehouse_sources/sources/sparkpost/api_inventory.md
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/canonical_descriptions.py b/products/warehouse_sources/sources/sparkpost/canonical_descriptions.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/canonical_descriptions.py
rename to products/warehouse_sources/sources/sparkpost/canonical_descriptions.py
index 645cc5a9721b..81e16d0eb5fb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/sparkpost/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
# Descriptions sourced from the public SparkPost API documentation. Keyed by the endpoint/schema
# name returned by `get_schemas`. Partial coverage is fine — anything omitted falls back to LLM
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/settings.py b/products/warehouse_sources/sources/sparkpost/settings.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/settings.py
rename to products/warehouse_sources/sources/sparkpost/settings.py
index a8af70e6b07e..d21ef2242d2a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/settings.py
+++ b/products/warehouse_sources/sources/sparkpost/settings.py
@@ -1,7 +1,7 @@
from dataclasses import dataclass, field
from typing import Literal, Optional
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType
PaginationStyle = Literal["cursor", "none"]
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/source.py b/products/warehouse_sources/sources/sparkpost/source.py
similarity index 85%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/source.py
rename to products/warehouse_sources/sources/sparkpost/source.py
index 2de74bfb7d44..2dd99d37eb95 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/source.py
+++ b/products/warehouse_sources/sources/sparkpost/source.py
@@ -1,37 +1,33 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
SourceFieldSelectConfig,
SourceFieldSelectConfigOption,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
WebhookSyncResult,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.sparkpost import (
- SparkPostSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost import sparkpost as api_client
-from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.settings import (
+from products.warehouse_sources.sources.sparkpost import sparkpost as api_client
+from products.warehouse_sources.sources.sparkpost._config import SparkPostSourceConfig
+from products.warehouse_sources.sources.sparkpost.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
LIMITED_RETENTION_ENDPOINTS,
@@ -40,15 +36,14 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.
WEBHOOK_RESOURCE_MAP,
WEBHOOK_SCHEMA_NAMES,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.sparkpost import (
+from products.warehouse_sources.sources.sparkpost.sparkpost import (
SparkPostResumeConfig,
sparkpost_source,
validate_credentials as validate_sparkpost_credentials,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@SourceRegistry.register
@@ -159,9 +154,7 @@ SparkPost runs independent US and EU stacks that do not share data — pick the
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.sparkpost.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
@@ -218,9 +211,7 @@ SparkPost runs independent US and EU stacks that do not share data — pick the
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.webhook_template import ( # noqa: PLC0415
- template,
- )
+ from products.warehouse_sources.sources.sparkpost.webhook_template import template # noqa: PLC0415
return template
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/sparkpost.py b/products/warehouse_sources/sources/sparkpost/sparkpost.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/sparkpost.py
rename to products/warehouse_sources/sources/sparkpost/sparkpost.py
index 1ea495dec174..06138f6250cc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/sparkpost.py
+++ b/products/warehouse_sources/sources/sparkpost/sparkpost.py
@@ -12,27 +12,23 @@ import structlog
from asgiref.sync import async_to_sync
from requests import Response, Session
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
+ BaseNextUrlPaginator,
ExternalWebhookInfo,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SinglePagePaginator,
+ SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
+ WebhookSourceManager,
WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
+ make_tracked_session,
rest_api_resource,
+ validate_via_probe,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- BaseNextUrlPaginator,
- SinglePagePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.settings import (
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sparkpost.settings import (
SPARKPOST_ENDPOINTS,
WEBHOOK_BATCH_KEY,
WEBHOOK_EVENT_GROUPING,
diff --git a/products/warehouse_sources/sources/sparkpost/tests/__init__.py b/products/warehouse_sources/sources/sparkpost/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/tests/test_sparkpost.py b/products/warehouse_sources/sources/sparkpost/tests/test_sparkpost.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/tests/test_sparkpost.py
rename to products/warehouse_sources/sources/sparkpost/tests/test_sparkpost.py
index d1f4b183bfbb..d7f11154d985 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/tests/test_sparkpost.py
+++ b/products/warehouse_sources/sources/sparkpost/tests/test_sparkpost.py
@@ -10,7 +10,7 @@ import pyarrow as pa
import requests
from requests import Response
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+from products.warehouse_sources.sources.sdk.testing import (
DriveResult,
RecordedRequest,
Script,
@@ -19,12 +19,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
always,
scripted_network,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.sparkpost import (
- SparkPostSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.settings import WEBHOOK_BATCH_KEY
-from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.source import SparkPostSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.sparkpost import (
+from products.warehouse_sources.sources.sparkpost._config import SparkPostSourceConfig
+from products.warehouse_sources.sources.sparkpost.settings import WEBHOOK_BATCH_KEY
+from products.warehouse_sources.sources.sparkpost.source import SparkPostSource
+from products.warehouse_sources.sources.sparkpost.sparkpost import (
SparkPostLinksPaginator,
SparkPostResumeConfig,
_format_from,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/tests/test_sparkpost_source.py b/products/warehouse_sources/sources/sparkpost/tests/test_sparkpost_source.py
similarity index 86%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/tests/test_sparkpost_source.py
rename to products/warehouse_sources/sources/sparkpost/tests/test_sparkpost_source.py
index 692c5e47f3d5..62fcc11bd6aa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/tests/test_sparkpost_source.py
+++ b/products/warehouse_sources/sources/sparkpost/tests/test_sparkpost_source.py
@@ -3,16 +3,10 @@ from typing import Any
import pytest
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- DriveResult,
- ScriptedResponse,
- SourceDriver,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.sparkpost import (
- SparkPostSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.settings import WEBHOOK_EVENT_TYPES
-from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.source import SparkPostSource
+from products.warehouse_sources.sources.sdk.testing import DriveResult, ScriptedResponse, SourceDriver
+from products.warehouse_sources.sources.sparkpost._config import SparkPostSourceConfig
+from products.warehouse_sources.sources.sparkpost.settings import WEBHOOK_EVENT_TYPES
+from products.warehouse_sources.sources.sparkpost.source import SparkPostSource
class TestSparkPostSource:
@@ -50,7 +44,7 @@ class TestSparkPostSource:
)
-API_CLIENT = "products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.source.api_client"
+API_CLIENT = "products.warehouse_sources.sources.sparkpost.source.api_client"
WEBHOOK_URL = "https://app.posthog.com/public/webhooks/abc"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/tests/test_sparkpost_webhook_template.py b/products/warehouse_sources/sources/sparkpost/tests/test_sparkpost_webhook_template.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/tests/test_sparkpost_webhook_template.py
rename to products/warehouse_sources/sources/sparkpost/tests/test_sparkpost_webhook_template.py
index 493fe7a8d8bc..937382d441e8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/tests/test_sparkpost_webhook_template.py
+++ b/products/warehouse_sources/sources/sparkpost/tests/test_sparkpost_webhook_template.py
@@ -3,16 +3,16 @@ import base64
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.settings import WEBHOOK_BATCH_KEY
-from products.warehouse_sources.backend.temporal.data_imports.sources.sparkpost.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
+from products.warehouse_sources.sources.sparkpost.settings import WEBHOOK_BATCH_KEY
+from products.warehouse_sources.sources.sparkpost.webhook_template import template
SOURCE_ID = "source_test_123"
AUTH_HEADER = "Basic " + base64.b64encode(b"posthog:s3cr3t").decode()
SCHEMA_MAPPING = {"message_event": "schema_events"}
-class TestSparkPostWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestSparkPostWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/webhook_template.py b/products/warehouse_sources/sources/sparkpost/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/webhook_template.py
rename to products/warehouse_sources/sources/sparkpost/webhook_template.py
index 8fcf1cbb290e..bbd5cf173058 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/sparkpost/webhook_template.py
+++ b/products/warehouse_sources/sources/sparkpost/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/sources/uppromote/__init__.py b/products/warehouse_sources/sources/uppromote/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/uppromote.py b/products/warehouse_sources/sources/uppromote/_config.py
similarity index 71%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/uppromote.py
rename to products/warehouse_sources/sources/uppromote/_config.py
index 1862984e689f..f11825dbe48e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/uppromote.py
+++ b/products/warehouse_sources/sources/uppromote/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/canonical_descriptions.py b/products/warehouse_sources/sources/uppromote/canonical_descriptions.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/canonical_descriptions.py
rename to products/warehouse_sources/sources/uppromote/canonical_descriptions.py
index 8d1e7c128dde..67f73f74b2c6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/uppromote/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"programs": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/settings.py b/products/warehouse_sources/sources/uppromote/settings.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/settings.py
rename to products/warehouse_sources/sources/uppromote/settings.py
index b35adfc54f09..73de95bf290f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/settings.py
+++ b/products/warehouse_sources/sources/uppromote/settings.py
@@ -1,10 +1,7 @@
from dataclasses import dataclass, field
from typing import Optional
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
- DependentEndpointConfig,
-)
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import DependentEndpointConfig, IncrementalField, IncrementalFieldType
UPPROMOTE_BASE_URL = "https://aff-api.uppromote.com/api/v2"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/source.py b/products/warehouse_sources/sources/uppromote/source.py
similarity index 84%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/source.py
rename to products/warehouse_sources/sources/uppromote/source.py
index df3e6df7b17e..debec75bacb9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/source.py
+++ b/products/warehouse_sources/sources/uppromote/source.py
@@ -1,42 +1,36 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
build_endpoint_schemas,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.uppromote import (
- UpPromoteSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.uppromote.settings import (
+from products.warehouse_sources.sources.uppromote._config import UpPromoteSourceConfig
+from products.warehouse_sources.sources.uppromote.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
RESOURCE_TO_UPPROMOTE_OBJECT_TYPE,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.uppromote.uppromote import (
+from products.warehouse_sources.sources.uppromote.uppromote import (
UpPromoteResumeConfig,
all_desired_webhook_events,
create_webhook as create_uppromote_webhook,
@@ -46,10 +40,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.uppromote.
uppromote_source,
validate_credentials as validate_uppromote_credentials,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@SourceRegistry.register
@@ -126,7 +119,7 @@ class UpPromoteSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.uppromote.webhook_template import template
+ from products.warehouse_sources.sources.uppromote.webhook_template import template
return template
@@ -135,9 +128,7 @@ class UpPromoteSource(
return RESOURCE_TO_UPPROMOTE_OBJECT_TYPE
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.uppromote.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.uppromote.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/uppromote/tests/__init__.py b/products/warehouse_sources/sources/uppromote/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/tests/test_uppromote.py b/products/warehouse_sources/sources/uppromote/tests/test_uppromote.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/tests/test_uppromote.py
rename to products/warehouse_sources/sources/uppromote/tests/test_uppromote.py
index 6fd46b59a111..31f57e68146e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/tests/test_uppromote.py
+++ b/products/warehouse_sources/sources/uppromote/tests/test_uppromote.py
@@ -6,8 +6,9 @@ from unittest.mock import AsyncMock, MagicMock, patch
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+from products.warehouse_sources.sources.sdk import WebhookSourceManager
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import (
RecordedRequest,
ScriptedResponse,
SourceDriver,
@@ -15,13 +16,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
route,
scripted_network,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.uppromote import (
- UpPromoteSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.uppromote.settings import UPPROMOTE_ENDPOINTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.uppromote.source import UpPromoteSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.uppromote.uppromote import (
+from products.warehouse_sources.sources.uppromote._config import UpPromoteSourceConfig
+from products.warehouse_sources.sources.uppromote.settings import UPPROMOTE_ENDPOINTS
+from products.warehouse_sources.sources.uppromote.source import UpPromoteSource
+from products.warehouse_sources.sources.uppromote.uppromote import (
UpPromoteResumeConfig,
_build_resource,
_format_datetime,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/tests/test_uppromote_source.py b/products/warehouse_sources/sources/uppromote/tests/test_uppromote_source.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/tests/test_uppromote_source.py
rename to products/warehouse_sources/sources/uppromote/tests/test_uppromote_source.py
index 41d7402635f1..ce8de75b5cac 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/tests/test_uppromote_source.py
+++ b/products/warehouse_sources/sources/uppromote/tests/test_uppromote_source.py
@@ -4,10 +4,10 @@ from unittest.mock import MagicMock, patch
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
-from products.warehouse_sources.backend.temporal.data_imports.sources.uppromote.source import UpPromoteSource
+from products.warehouse_sources.sources.sdk import SourceInputs
+from products.warehouse_sources.sources.uppromote.source import UpPromoteSource
-SOURCE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.uppromote.source"
+SOURCE_MODULE = "products.warehouse_sources.sources.uppromote.source"
def _make_inputs(**overrides: Any) -> SourceInputs:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/uppromote.py b/products/warehouse_sources/sources/uppromote/uppromote.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/uppromote.py
rename to products/warehouse_sources/sources/uppromote/uppromote.py
index dffbeb12cce1..5f8ba80b2906 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/uppromote.py
+++ b/products/warehouse_sources/sources/uppromote/uppromote.py
@@ -8,33 +8,25 @@ import requests
from asgiref.sync import async_to_sync
from dateutil import parser
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
+ ClientConfig,
+ DependentEndpointConfig,
+ EndpointResource,
ExternalWebhookInfo,
+ PageNumberPaginator,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
+ WebhookSourceManager,
WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
- DependentEndpointConfig,
build_dependent_resource,
+ make_tracked_session,
+ rest_api_resource,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- PageNumberPaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- ClientConfig,
- EndpointResource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.uppromote.settings import (
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.uppromote.settings import (
UPPROMOTE_BASE_URL,
UPPROMOTE_ENDPOINTS,
UPPROMOTE_OBJECT_TYPE_TO_EVENTS,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/webhook_template.py b/products/warehouse_sources/sources/uppromote/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/webhook_template.py
rename to products/warehouse_sources/sources/uppromote/webhook_template.py
index 65d45e6327c3..f47cd3df17e2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/uppromote/webhook_template.py
+++ b/products/warehouse_sources/sources/uppromote/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
# UpPromote signs webhooks with HMAC-SHA256 over the raw request body using the subscription's
# secret key, sent as X-UpPromote-Signature (hex digest, no timestamp). The payload is the bare
diff --git a/products/warehouse_sources/sources/webflow/__init__.py b/products/warehouse_sources/sources/webflow/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/webflow.py b/products/warehouse_sources/sources/webflow/_config.py
similarity index 73%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/webflow.py
rename to products/warehouse_sources/sources/webflow/_config.py
index fd0fea9ed2b2..d22f9daf3e13 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/webflow.py
+++ b/products/warehouse_sources/sources/webflow/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/canonical_descriptions.py b/products/warehouse_sources/sources/webflow/canonical_descriptions.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/webflow/canonical_descriptions.py
rename to products/warehouse_sources/sources/webflow/canonical_descriptions.py
index 4c3ad98116e0..dc65d33f56d6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/webflow/canonical_descriptions.py
@@ -7,9 +7,7 @@ user-defined CMS collections discovered at sync time, so they intentionally have
and fall back to LLM enrichment. Columns absent here also fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"sites": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/settings.py b/products/warehouse_sources/sources/webflow/settings.py
similarity index 100%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/webflow/settings.py
rename to products/warehouse_sources/sources/webflow/settings.py
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/source.py b/products/warehouse_sources/sources/webflow/source.py
similarity index 89%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/webflow/source.py
rename to products/warehouse_sources/sources/webflow/source.py
index f04d62857ba0..c87f2df9c1d7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/source.py
+++ b/products/warehouse_sources/sources/webflow/source.py
@@ -2,41 +2,37 @@ from typing import TYPE_CHECKING, Any, Optional, cast
import structlog
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.webflow import (
- WebflowSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.webflow.settings import (
+from products.warehouse_sources.sources.webflow._config import WebflowSourceConfig
+from products.warehouse_sources.sources.webflow.settings import (
COLLECTION_SCHEMA_PREFIX,
SCHEMA_TO_WEBHOOK_EVENTS,
STATIC_ENDPOINTS,
WEBHOOK_RESOURCE_MAP,
WEBHOOK_SCHEMA_NAMES,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.webflow.webflow import (
+from products.warehouse_sources.sources.webflow.webflow import (
WebflowResumeConfig,
create_webhook as create_webflow_webhook,
delete_webhook as delete_webflow_webhook,
@@ -45,10 +41,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.webflow.we
validate_credentials as validate_webflow_credentials,
webflow_source,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
logger = structlog.get_logger(__name__)
@@ -78,9 +73,7 @@ class WebflowSource(
return ExternalDataSourceType.WEBFLOW
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.webflow.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.webflow.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
@@ -178,7 +171,7 @@ class WebflowSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.webflow.webhook_template import template
+ from products.warehouse_sources.sources.webflow.webhook_template import template
return template
diff --git a/products/warehouse_sources/sources/webflow/tests/__init__.py b/products/warehouse_sources/sources/webflow/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/tests/test_webflow.py b/products/warehouse_sources/sources/webflow/tests/test_webflow.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/webflow/tests/test_webflow.py
rename to products/warehouse_sources/sources/webflow/tests/test_webflow.py
index 58e25430b771..1f42ba056da7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/tests/test_webflow.py
+++ b/products/warehouse_sources/sources/webflow/tests/test_webflow.py
@@ -7,25 +7,19 @@ from unittest.mock import AsyncMock, MagicMock, patch
import requests
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+from products.warehouse_sources.sources.sdk import ResumableSourceManager, WebhookSourceManager
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import (
RecordedRequest,
ScriptedResponse,
SourceDriver,
always,
scripted_network,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.webflow import (
- WebflowSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.webflow.settings import (
- ALL_WEBHOOK_EVENTS,
- WEBFLOW_ENDPOINTS,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.webflow.source import WebflowSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.webflow.webflow import (
+from products.warehouse_sources.sources.webflow._config import WebflowSourceConfig
+from products.warehouse_sources.sources.webflow.settings import ALL_WEBHOOK_EVENTS, WEBFLOW_ENDPOINTS
+from products.warehouse_sources.sources.webflow.source import WebflowSource
+from products.warehouse_sources.sources.webflow.webflow import (
WebflowResumeConfig,
_extract_items,
_resolve_collection_id,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/tests/test_webflow_source.py b/products/warehouse_sources/sources/webflow/tests/test_webflow_source.py
similarity index 88%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/webflow/tests/test_webflow_source.py
rename to products/warehouse_sources/sources/webflow/tests/test_webflow_source.py
index c516f9c393e8..ed0705f107ba 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/tests/test_webflow_source.py
+++ b/products/warehouse_sources/sources/webflow/tests/test_webflow_source.py
@@ -2,16 +2,12 @@ from unittest.mock import MagicMock, patch
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.webflow import (
- WebflowSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.webflow.settings import STATIC_ENDPOINTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.webflow.source import WebflowSource
-
-SOURCE_MODULE = "products.warehouse_sources.backend.temporal.data_imports.sources.webflow.source"
+from products.warehouse_sources.sources.sdk import ResumableSourceManager, SourceInputs, WebhookSourceManager
+from products.warehouse_sources.sources.webflow._config import WebflowSourceConfig
+from products.warehouse_sources.sources.webflow.settings import STATIC_ENDPOINTS
+from products.warehouse_sources.sources.webflow.source import WebflowSource
+
+SOURCE_MODULE = "products.warehouse_sources.sources.webflow.source"
def _config() -> WebflowSourceConfig:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/tests/test_webflow_webhook_template.py b/products/warehouse_sources/sources/webflow/tests/test_webflow_webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/webflow/tests/test_webflow_webhook_template.py
rename to products/warehouse_sources/sources/webflow/tests/test_webflow_webhook_template.py
index f02e6aeb7d46..9d2fa5cd25cd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/tests/test_webflow_webhook_template.py
+++ b/products/warehouse_sources/sources/webflow/tests/test_webflow_webhook_template.py
@@ -5,8 +5,8 @@ import hashlib
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.webflow.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
+from products.warehouse_sources.sources.webflow.webhook_template import template
SOURCE_ID = "source_test_123"
NEW_ORDER_SECRET = "2b4acfd1c5518bf03c73a4889d197d77251353857c22694bf150b9e3402ba15f"
@@ -18,7 +18,7 @@ def _now_ms() -> int:
return int(time.time() * 1000)
-class TestWebflowWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestWebflowWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/webflow.py b/products/warehouse_sources/sources/webflow/webflow.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/webflow/webflow.py
rename to products/warehouse_sources/sources/webflow/webflow.py
index db939c0b99cc..36d37ee47680 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/webflow.py
+++ b/products/warehouse_sources/sources/webflow/webflow.py
@@ -8,27 +8,23 @@ import pyarrow as pa
import requests
from asgiref.sync import async_to_sync
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- WebhookCreationResult,
- WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
+from products.warehouse_sources.sources.sdk import (
Endpoint,
EndpointResource,
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
+ ExternalWebhookInfo,
OffsetPaginator,
+ RESTAPIConfig,
+ ResumableSourceManager,
SinglePagePaginator,
+ SourceResponse,
+ WebhookCreationResult,
+ WebhookDeletionResult,
+ WebhookSourceManager,
+ make_tracked_session,
+ rest_api_resource,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.webflow.settings import (
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.webflow.settings import (
ALL_WEBHOOK_EVENTS,
COLLECTION_SCHEMA_PREFIX,
DEFAULT_PAGE_SIZE,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/webhook_template.py b/products/warehouse_sources/sources/webflow/webhook_template.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/webflow/webhook_template.py
rename to products/warehouse_sources/sources/webflow/webhook_template.py
index 8e9d71a8f690..05b711daf3f7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/webflow/webhook_template.py
+++ b/products/warehouse_sources/sources/webflow/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/sources/whop/__init__.py b/products/warehouse_sources/sources/whop/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/whop.py b/products/warehouse_sources/sources/whop/_config.py
similarity index 73%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/whop.py
rename to products/warehouse_sources/sources/whop/_config.py
index 23be5a19228e..8d62410eec20 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/whop.py
+++ b/products/warehouse_sources/sources/whop/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/canonical_descriptions.py b/products/warehouse_sources/sources/whop/canonical_descriptions.py
similarity index 99%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/whop/canonical_descriptions.py
rename to products/warehouse_sources/sources/whop/canonical_descriptions.py
index 5e1402030389..553f6289108c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/whop/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
# Table and column descriptions transcribed from Whop's stable REST API reference
# (https://docs.whop.com/api-reference). Columns the vendor does not describe are left out and
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/settings.py b/products/warehouse_sources/sources/whop/settings.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/whop/settings.py
rename to products/warehouse_sources/sources/whop/settings.py
index 280e7d310c22..3d0f71a8f82c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/settings.py
+++ b/products/warehouse_sources/sources/whop/settings.py
@@ -1,8 +1,4 @@
-from posthog.dataclasses import frozen
-
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SortMode
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.sources.sdk import IncrementalField, SortMode, frozen, incremental_field
BASE_URL = "https://api.whop.com/api/v1"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/source.py b/products/warehouse_sources/sources/whop/source.py
similarity index 88%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/whop/source.py
rename to products/warehouse_sources/sources/whop/source.py
index 5ce02d9d160f..ad1626102423 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/source.py
+++ b/products/warehouse_sources/sources/whop/source.py
@@ -1,37 +1,33 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.sdk import (
+ UNVERSIONED_API_VERSION,
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- UNVERSIONED_API_VERSION,
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
build_endpoint_schemas,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.whop import WhopSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.whop import whop as api_client
-from products.warehouse_sources.backend.temporal.data_imports.sources.whop.settings import (
+from products.warehouse_sources.sources.whop import whop as api_client
+from products.warehouse_sources.sources.whop._config import WhopSourceConfig
+from products.warehouse_sources.sources.whop.settings import (
INCREMENTAL_FIELDS,
MERGE_ONLY_ENDPOINTS,
RESOURCE_TO_EVENT_PREFIX,
@@ -43,11 +39,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.whop.setti
endpoints_for,
webhook_schema_names_for,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.whop.whop import WhopResumeConfig
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.whop.whop import WhopResumeConfig
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@SourceRegistry.register
@@ -192,9 +187,7 @@ class WhopSource(
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.whop.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.whop.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
@@ -252,7 +245,7 @@ class WhopSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.whop.webhook_template import template
+ from products.warehouse_sources.sources.whop.webhook_template import template
return template
diff --git a/products/warehouse_sources/sources/whop/tests/__init__.py b/products/warehouse_sources/sources/whop/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/tests/test_whop.py b/products/warehouse_sources/sources/whop/tests/test_whop.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/whop/tests/test_whop.py
rename to products/warehouse_sources/sources/whop/tests/test_whop.py
index 7b972c922bc6..7979efdea530 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/tests/test_whop.py
+++ b/products/warehouse_sources/sources/whop/tests/test_whop.py
@@ -6,23 +6,18 @@ import pytest
from requests import Response
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import (
RecordedRequest,
ScriptedResponse,
SourceDriver,
always,
scripted_network,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.whop import WhopSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.whop.settings import (
- BASE_URL,
- ENDPOINTS,
- PAGE_SIZE,
- WHOP_ENDPOINTS,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.whop.source import WhopSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.whop.whop import (
+from products.warehouse_sources.sources.whop._config import WhopSourceConfig
+from products.warehouse_sources.sources.whop.settings import BASE_URL, ENDPOINTS, PAGE_SIZE, WHOP_ENDPOINTS
+from products.warehouse_sources.sources.whop.source import WhopSource
+from products.warehouse_sources.sources.whop.whop import (
WhopCursorPaginator,
WhopResumeConfig,
_list_params,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/tests/test_whop_source.py b/products/warehouse_sources/sources/whop/tests/test_whop_source.py
similarity index 92%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/whop/tests/test_whop_source.py
rename to products/warehouse_sources/sources/whop/tests/test_whop_source.py
index bef03ab6c259..23f3281551b0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/tests/test_whop_source.py
+++ b/products/warehouse_sources/sources/whop/tests/test_whop_source.py
@@ -1,12 +1,12 @@
import pytest
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import error_message_matches
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.whop import WhopSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.whop.settings import SCHEMA_TO_WEBHOOK_EVENTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.whop.source import WhopSource
+from products.warehouse_sources.sources.sdk.testing import error_message_matches
+from products.warehouse_sources.sources.whop._config import WhopSourceConfig
+from products.warehouse_sources.sources.whop.settings import SCHEMA_TO_WEBHOOK_EVENTS
+from products.warehouse_sources.sources.whop.source import WhopSource
-API_CLIENT_PATCH = "products.warehouse_sources.backend.temporal.data_imports.sources.whop.source.api_client"
+API_CLIENT_PATCH = "products.warehouse_sources.sources.whop.source.api_client"
WEBHOOK_URL = "https://ph.example/webhook"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/webhook_template.py b/products/warehouse_sources/sources/whop/webhook_template.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/whop/webhook_template.py
rename to products/warehouse_sources/sources/whop/webhook_template.py
index 92316986982a..37383f739e6c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/webhook_template.py
+++ b/products/warehouse_sources/sources/whop/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
# Whop follows the Standard Webhooks spec: `webhook-id`, `webhook-timestamp` (Unix seconds) and
# `webhook-signature` (a space-separated list of `v<version>,<base64 HMAC-SHA256>` entries) signing
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/whop.py b/products/warehouse_sources/sources/whop/whop.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/whop/whop.py
rename to products/warehouse_sources/sources/whop/whop.py
index f00c9de28428..c071f79d16e6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/whop/whop.py
+++ b/products/warehouse_sources/sources/whop/whop.py
@@ -9,26 +9,22 @@ import pyarrow as pa
from asgiref.sync import async_to_sync
from requests import Response, Session
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
ExternalWebhookInfo,
+ JSONResponseCursorPaginator,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
+ WebhookSourceManager,
WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
+ make_tracked_session,
rest_api_resource,
+ validate_via_probe,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- JSONResponseCursorPaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import validate_via_probe
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.whop.settings import (
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.whop.settings import (
API_VERSION_HEADER,
BASE_URL,
COMPANY_FILTER_PARAM,
diff --git a/products/warehouse_sources/sources/woocommerce/__init__.py b/products/warehouse_sources/sources/woocommerce/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/woocommerce.py b/products/warehouse_sources/sources/woocommerce/_config.py
similarity index 75%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/woocommerce.py
rename to products/warehouse_sources/sources/woocommerce/_config.py
index fae6e953a136..259d8130cae3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/woocommerce.py
+++ b/products/warehouse_sources/sources/woocommerce/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/canonical_descriptions.py b/products/warehouse_sources/sources/woocommerce/canonical_descriptions.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/canonical_descriptions.py
rename to products/warehouse_sources/sources/woocommerce/canonical_descriptions.py
index 9c0e2b36d63d..81747a0c3dd5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/woocommerce/canonical_descriptions.py
@@ -5,9 +5,7 @@ Keyed by the endpoint names in `settings.py` `ENDPOINTS`, which match the `Exter
of a synced WooCommerce table. Columns absent here fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"products": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/settings.py b/products/warehouse_sources/sources/woocommerce/settings.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/settings.py
rename to products/warehouse_sources/sources/woocommerce/settings.py
index 50623e61efcd..701bc814d4f5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/settings.py
+++ b/products/warehouse_sources/sources/woocommerce/settings.py
@@ -1,4 +1,4 @@
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType
# Resources a user is likely to want from a WooCommerce store. Each maps to a
# top-level WooCommerce REST API v3 collection endpoint (relative to
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/source.py b/products/warehouse_sources/sources/woocommerce/source.py
similarity index 88%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/source.py
rename to products/warehouse_sources/sources/woocommerce/source.py
index a356c82eec2c..2d440b9212ac 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/source.py
+++ b/products/warehouse_sources/sources/woocommerce/source.py
@@ -2,44 +2,38 @@ from typing import TYPE_CHECKING, Optional, cast
from asgiref.sync import async_to_sync
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
+ WebhookSourceManager,
build_endpoint_schemas,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.woocommerce import (
- WooCommerceSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.settings import (
+from products.warehouse_sources.sources.woocommerce._config import WooCommerceSourceConfig
+from products.warehouse_sources.sources.woocommerce.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
PARTITION_FIELDS,
SCHEMA_TO_WEBHOOK_RESOURCE,
WEBHOOK_SCHEMA_NAMES,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.woocommerce import (
+from products.warehouse_sources.sources.woocommerce.woocommerce import (
WooCommerceResumeConfig,
create_webhook as create_woocommerce_webhook,
delete_webhook as delete_woocommerce_webhook,
@@ -48,10 +42,9 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.woocommerc
webhook_table_transformer,
woocommerce_source,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
WEBHOOK_SETUP_CAPTION = (
"PostHog registers a webhook on your store for each of the product, order, coupon and customer "
@@ -90,9 +83,7 @@ class WooCommerceSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.webhook_template import ( # noqa: PLC0415
- template,
- )
+ from products.warehouse_sources.sources.woocommerce.webhook_template import template # noqa: PLC0415
return template
@@ -104,9 +95,7 @@ class WooCommerceSource(
return SCHEMA_TO_WEBHOOK_RESOURCE
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.woocommerce.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/woocommerce/tests/__init__.py b/products/warehouse_sources/sources/woocommerce/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/tests/test_woocommerce.py b/products/warehouse_sources/sources/woocommerce/tests/test_woocommerce.py
similarity index 88%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/tests/test_woocommerce.py
rename to products/warehouse_sources/sources/woocommerce/tests/test_woocommerce.py
index ae2781a26d88..2a0e3e17fe4c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/tests/test_woocommerce.py
+++ b/products/warehouse_sources/sources/woocommerce/tests/test_woocommerce.py
@@ -6,22 +6,13 @@ from unittest.mock import MagicMock, patch
from requests import Request
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import TrackedHTTPAdapter
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import (
- HostResolution,
- TemporaryHostResolutionError,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.woocommerce import (
- WooCommerceSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.settings import WEBHOOK_TOPICS
-from products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.source import WooCommerceSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.woocommerce import (
+from products.warehouse_sources.sources.sdk import TemporaryHostResolutionError, TrackedHTTPAdapter
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import HostResolution, ScriptedResponse, SourceDriver
+from products.warehouse_sources.sources.woocommerce._config import WooCommerceSourceConfig
+from products.warehouse_sources.sources.woocommerce.settings import WEBHOOK_TOPICS
+from products.warehouse_sources.sources.woocommerce.source import WooCommerceSource
+from products.warehouse_sources.sources.woocommerce.woocommerce import (
DEFAULT_PER_PAGE,
WOOCOMMERCE_USER_AGENT,
WooCommerceAuth,
@@ -192,28 +183,22 @@ class TestWooCommerceAuth:
class TestValidateCredentials:
@pytest.mark.parametrize("status_code", [403, 404])
def test_returns_status_code(self, status_code: int) -> None:
- with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.woocommerce._make_guarded_session"
- ) as MockSession:
+ with patch("products.warehouse_sources.sources.woocommerce.woocommerce._make_guarded_session") as MockSession:
MockSession.return_value.get.return_value = MagicMock(status_code=status_code)
assert validate_credentials("https://example.com", "ck", "cs", 123) == status_code
def test_returns_none_on_connection_error(self) -> None:
- with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.woocommerce._make_guarded_session"
- ) as MockSession:
+ with patch("products.warehouse_sources.sources.woocommerce.woocommerce._make_guarded_session") as MockSession:
MockSession.return_value.get.side_effect = Exception("boom")
assert validate_credentials("https://example.com", "ck", "cs", 123) is None
def test_unsafe_host_short_circuits_without_request(self) -> None:
with (
patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.woocommerce._is_host_safe",
+ "products.warehouse_sources.sources.woocommerce.woocommerce._is_host_safe",
return_value=(False, "blocked"),
),
- patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.woocommerce._make_guarded_session"
- ) as MockSession,
+ patch("products.warehouse_sources.sources.woocommerce.woocommerce._make_guarded_session") as MockSession,
):
assert validate_credentials("https://169.254.169.254", "ck", "cs", 123) is None
MockSession.return_value.get.assert_not_called()
@@ -246,7 +231,7 @@ class TestHostGuardedAdapter:
adapter = _HostGuardedAdapter(team_id=123)
with (
patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.woocommerce.resolve_safe_host",
+ "products.warehouse_sources.sources.woocommerce.woocommerce.resolve_safe_host",
**resolver_outcome,
),
patch.object(TrackedHTTPAdapter, "send") as mock_super_send,
@@ -262,7 +247,7 @@ class TestHostGuardedAdapter:
sentinel = MagicMock()
with (
patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.woocommerce.resolve_safe_host",
+ "products.warehouse_sources.sources.woocommerce.woocommerce.resolve_safe_host",
return_value=HostResolution(connect_host="93.184.216.34", error=None),
),
patch.object(TrackedHTTPAdapter, "send", return_value=sentinel) as mock_super_send,
@@ -278,7 +263,7 @@ class TestSourceHostGuard:
config = WooCommerceSourceConfig(store_url="https://169.254.169.254", consumer_key="ck", consumer_secret="cs")
with patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.woocommerce.resolve_safe_host",
+ "products.warehouse_sources.sources.woocommerce.woocommerce.resolve_safe_host",
return_value=HostResolution(connect_host=None, error="Hosts with internal IP addresses are not allowed"),
):
result = SourceDriver(WooCommerceSource(), config).run("products", [])
@@ -308,7 +293,7 @@ class TestWebhookManagement:
def _patch(self, session: MagicMock) -> Any:
return patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.woocommerce._make_guarded_session",
+ "products.warehouse_sources.sources.woocommerce.woocommerce._make_guarded_session",
return_value=session,
)
@@ -419,12 +404,10 @@ class TestWebhookManagement:
def test_unsafe_host_never_reaches_the_network(self, operation: Any, resolver_outcome: dict[str, Any]) -> None:
with (
patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.woocommerce.resolve_safe_host",
+ "products.warehouse_sources.sources.woocommerce.woocommerce.resolve_safe_host",
**resolver_outcome,
),
- patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.woocommerce._make_guarded_session"
- ) as MockSession,
+ patch("products.warehouse_sources.sources.woocommerce.woocommerce._make_guarded_session") as MockSession,
):
result = operation("https://169.254.169.254", "ck", "cs", 123, "https://ph.test/hook")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/tests/test_woocommerce_source.py b/products/warehouse_sources/sources/woocommerce/tests/test_woocommerce_source.py
similarity index 83%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/tests/test_woocommerce_source.py
rename to products/warehouse_sources/sources/woocommerce/tests/test_woocommerce_source.py
index 3f671776e0c4..652a53aab08c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/tests/test_woocommerce_source.py
+++ b/products/warehouse_sources/sources/woocommerce/tests/test_woocommerce_source.py
@@ -1,22 +1,16 @@
import pytest
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.woocommerce import (
- WooCommerceSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.settings import (
+from products.warehouse_sources.sources.sdk import ResumableSourceManager, WebhookSourceManager
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver
+from products.warehouse_sources.sources.woocommerce._config import WooCommerceSourceConfig
+from products.warehouse_sources.sources.woocommerce.settings import (
INCREMENTAL_FIELDS,
SCHEMA_TO_WEBHOOK_RESOURCE,
WEBHOOK_SCHEMA_NAMES,
WEBHOOK_TOPICS,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.source import WooCommerceSource
+from products.warehouse_sources.sources.woocommerce.source import WooCommerceSource
INCREMENTAL_ENDPOINTS = set(INCREMENTAL_FIELDS.keys())
@@ -54,9 +48,7 @@ class TestWooCommerceSource:
(None, None, False, "Couldn't reach your WooCommerce store"), # connection error
],
)
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.source.validate_woocommerce_credentials"
- )
+ @mock.patch("products.warehouse_sources.sources.woocommerce.source.validate_woocommerce_credentials")
def test_validate_credentials(self, mock_validate, status, schema_name, expected_valid, expected_message_fragment):
mock_validate.return_value = status
@@ -101,9 +93,7 @@ class TestWooCommerceSource:
def test_webhook_management_passes_the_store_credentials_through(self, method_name, transport_name):
# The store URL and the key/secret pair are positional; swapping two of them would send
# the consumer key as the secret and 401 on every store.
- with mock.patch(
- f"products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.source.{transport_name}"
- ) as mock_transport:
+ with mock.patch(f"products.warehouse_sources.sources.woocommerce.source.{transport_name}") as mock_transport:
getattr(self.source, method_name)(self.config, "https://ph.test/hook", self.team_id)
mock_transport.assert_called_once_with(
@@ -123,9 +113,7 @@ class TestWooCommerceSource:
manager = mock.MagicMock(spec=ResumableSourceManager)
with (
- mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.source.woocommerce_source"
- ) as mock_source,
+ mock.patch("products.warehouse_sources.sources.woocommerce.source.woocommerce_source") as mock_source,
mock.patch.object(WebhookSourceManager, "webhook_enabled", new=mock.AsyncMock(return_value=True)),
mock.patch.object(WebhookSourceManager, "get_items") as mock_get_items,
):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/tests/test_woocommerce_webhook_template.py b/products/warehouse_sources/sources/woocommerce/tests/test_woocommerce_webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/tests/test_woocommerce_webhook_template.py
rename to products/warehouse_sources/sources/woocommerce/tests/test_woocommerce_webhook_template.py
index 2527e4689bd0..2e14b81d650a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/tests/test_woocommerce_webhook_template.py
+++ b/products/warehouse_sources/sources/woocommerce/tests/test_woocommerce_webhook_template.py
@@ -5,8 +5,8 @@ import hashlib
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
+from products.warehouse_sources.sources.woocommerce.webhook_template import template
SOURCE_ID = "source_test_123"
SIGNING_SECRET = "cQ8oXhZ0mF3jWl2pR7tYb5NvA1sKdE6g"
@@ -18,7 +18,7 @@ SCHEMA_MAPPING = {
}
-class TestWooCommerceWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestWooCommerceWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/webhook_template.py b/products/warehouse_sources/sources/woocommerce/webhook_template.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/webhook_template.py
rename to products/warehouse_sources/sources/woocommerce/webhook_template.py
index 698ae10e5617..f174ba671a85 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/webhook_template.py
+++ b/products/warehouse_sources/sources/woocommerce/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/woocommerce.py b/products/warehouse_sources/sources/woocommerce/woocommerce.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/woocommerce.py
rename to products/warehouse_sources/sources/woocommerce/woocommerce.py
index 538116a91f10..e323c041dfae 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/woocommerce/woocommerce.py
+++ b/products/warehouse_sources/sources/woocommerce/woocommerce.py
@@ -8,34 +8,24 @@ from urllib.parse import urlparse
import pyarrow as pa
from requests import PreparedRequest, Request, RequestException, Response, Session
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
+ DEFAULT_RETRY,
+ AuthConfigBase,
+ BasePaginator,
+ EndpointResource,
ExternalWebhookInfo,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ TemporaryHostResolutionError,
+ TrackedHTTPAdapter,
WebhookCreationResult,
WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import (
- DEFAULT_RETRY,
- TrackedHTTPAdapter,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import (
- TemporaryHostResolutionError,
_is_host_safe,
resolve_safe_host,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
rest_api_resource,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.auth import AuthConfigBase
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import EndpointResource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.woocommerce.settings import (
- ENDPOINT_PATHS,
- INCREMENTAL_FIELDS,
- WEBHOOK_TOPICS,
-)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.woocommerce.settings import ENDPOINT_PATHS, INCREMENTAL_FIELDS, WEBHOOK_TOPICS
# All WooCommerce REST API v3 endpoints hang off this path on the store domain.
WOOCOMMERCE_API_BASE_PATH = "/wp-json/wc/v3"
diff --git a/products/warehouse_sources/sources/workos/__init__.py b/products/warehouse_sources/sources/workos/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/workos.py b/products/warehouse_sources/sources/workos/_config.py
similarity index 71%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/workos.py
rename to products/warehouse_sources/sources/workos/_config.py
index 9a037493de78..3d90645b1464 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/workos.py
+++ b/products/warehouse_sources/sources/workos/_config.py
@@ -1,7 +1,7 @@
# This file is automatically generated from `SourceRegistry.get_all_sources()`
# Do not edit manually - run `pnpm generate:source-configs` to regenerate.
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/canonical_descriptions.py b/products/warehouse_sources/sources/workos/canonical_descriptions.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/workos/canonical_descriptions.py
rename to products/warehouse_sources/sources/workos/canonical_descriptions.py
index 11d8945f1137..6bfa43ddcee3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/workos/canonical_descriptions.py
@@ -5,9 +5,7 @@ endpoint names in `settings.py` `WORKOS_ENDPOINTS`, which match the `ExternalDat
synced WorkOS table. Columns absent here fall back to LLM enrichment.
"""
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
# Fields shared by most WorkOS objects; merged into each entry so we don't repeat them.
_COMMON_COLUMNS = {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/settings.py b/products/warehouse_sources/sources/workos/settings.py
similarity index 96%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/workos/settings.py
rename to products/warehouse_sources/sources/workos/settings.py
index e34e172d241b..8025b0494abb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/settings.py
+++ b/products/warehouse_sources/sources/workos/settings.py
@@ -1,8 +1,6 @@
from dataclasses import dataclass, field
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
- DependentEndpointConfig,
-)
+from products.warehouse_sources.sources.sdk import DependentEndpointConfig
@dataclass(frozen=True)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/source.py b/products/warehouse_sources/sources/workos/source.py
similarity index 83%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/workos/source.py
rename to products/warehouse_sources/sources/workos/source.py
index 5239bc6801f3..4333ab4b5d32 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/source.py
+++ b/products/warehouse_sources/sources/workos/source.py
@@ -1,41 +1,34 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
+from products.warehouse_sources.sources.sdk import WebhookSourceManager, detect
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
WebhookSyncResult,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import SourceSchema
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.workos import WorkOSSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.workos.settings import (
- ALL_WEBHOOK_EVENTS,
- ENDPOINTS,
- WEBHOOK_SCHEMA_NAMES,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.workos.workos import (
+from products.warehouse_sources.sources.workos._config import WorkOSSourceConfig
+from products.warehouse_sources.sources.workos.settings import ALL_WEBHOOK_EVENTS, ENDPOINTS, WEBHOOK_SCHEMA_NAMES
+from products.warehouse_sources.sources.workos.workos import (
WorkOSResumeConfig,
create_webhook as create_workos_webhook,
delete_webhook as delete_workos_webhook,
@@ -44,7 +37,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.workos.wor
validate_credentials as validate_workos_credentials,
workos_source,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
@SourceRegistry.register
@@ -83,7 +75,7 @@ class WorkOSSource(
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.workos.webhook_template import template
+ from products.warehouse_sources.sources.workos.webhook_template import template
return template
@@ -142,9 +134,7 @@ The key starts with `sk_`.
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.workos.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.workos.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
diff --git a/products/warehouse_sources/sources/workos/tests/__init__.py b/products/warehouse_sources/sources/workos/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/tests/test_workos.py b/products/warehouse_sources/sources/workos/tests/test_workos.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/workos/tests/test_workos.py
rename to products/warehouse_sources/sources/workos/tests/test_workos.py
index 5c77ed62c2c5..72158761e3ff 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/tests/test_workos.py
+++ b/products/warehouse_sources/sources/workos/tests/test_workos.py
@@ -5,7 +5,7 @@ from typing import Any
import pytest
from unittest.mock import MagicMock
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+from products.warehouse_sources.sources.sdk.testing import (
DriveResult,
RecordedRequest,
ScriptedResponse,
@@ -13,10 +13,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.tes
always,
scripted_network,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.workos import WorkOSSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.workos.settings import ENDPOINTS, WORKOS_ENDPOINTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.workos.source import WorkOSSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.workos.workos import (
+from products.warehouse_sources.sources.workos._config import WorkOSSourceConfig
+from products.warehouse_sources.sources.workos.settings import ENDPOINTS, WORKOS_ENDPOINTS
+from products.warehouse_sources.sources.workos.source import WorkOSSource
+from products.warehouse_sources.sources.workos.workos import (
WorkOSPaginator,
WorkOSResumeConfig,
create_webhook,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/tests/test_workos_source.py b/products/warehouse_sources/sources/workos/tests/test_workos_source.py
similarity index 75%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/workos/tests/test_workos_source.py
rename to products/warehouse_sources/sources/workos/tests/test_workos_source.py
index 2e1f6c7a6065..3c86c39fbce4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/tests/test_workos_source.py
+++ b/products/warehouse_sources/sources/workos/tests/test_workos_source.py
@@ -1,8 +1,8 @@
import pyarrow as pa
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.workos import WorkOSSourceConfig
-from products.warehouse_sources.backend.temporal.data_imports.sources.workos.source import WorkOSSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.workos.workos import _webhook_table_transformer
+from products.warehouse_sources.sources.workos._config import WorkOSSourceConfig
+from products.warehouse_sources.sources.workos.source import WorkOSSource
+from products.warehouse_sources.sources.workos.workos import _webhook_table_transformer
class TestWorkOSSource:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/tests/test_workos_webhook_template.py b/products/warehouse_sources/sources/workos/tests/test_workos_webhook_template.py
similarity index 92%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/workos/tests/test_workos_webhook_template.py
rename to products/warehouse_sources/sources/workos/tests/test_workos_webhook_template.py
index f33eaf652c6f..e3c24b5fff58 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/tests/test_workos_webhook_template.py
+++ b/products/warehouse_sources/sources/workos/tests/test_workos_webhook_template.py
@@ -5,12 +5,12 @@ import hashlib
from parameterized import parameterized
-from products.cdp.backend.templates.helpers import BaseHogFunctionTemplateTest
-from products.warehouse_sources.backend.temporal.data_imports.sources.workos.settings import WEBHOOK_EVENT_TO_SCHEMA
-from products.warehouse_sources.backend.temporal.data_imports.sources.workos.webhook_template import template
+from products.warehouse_sources.sources.sdk.testing import BaseHogFunctionTemplateSimpleTest
+from products.warehouse_sources.sources.workos.settings import WEBHOOK_EVENT_TO_SCHEMA
+from products.warehouse_sources.sources.workos.webhook_template import template
-class TestWorkOSWarehouseWebhookTemplate(BaseHogFunctionTemplateTest):
+class TestWorkOSWarehouseWebhookTemplate(BaseHogFunctionTemplateSimpleTest):
template = template
def createHogGlobals(self, globals=None) -> dict:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/webhook_template.py b/products/warehouse_sources/sources/workos/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/workos/webhook_template.py
rename to products/warehouse_sources/sources/workos/webhook_template.py
index 57d709ad2c72..95f77a21572b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/webhook_template.py
+++ b/products/warehouse_sources/sources/workos/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/workos.py b/products/warehouse_sources/sources/workos/workos.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/workos/workos.py
rename to products/warehouse_sources/sources/workos/workos.py
index f288ad0d42a1..4c95e697923a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/workos/workos.py
+++ b/products/warehouse_sources/sources/workos/workos.py
@@ -8,36 +8,26 @@ from asgiref.sync import async_to_sync
from requests import Request, Response
from requests.exceptions import RequestException
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
+ BasePaginator,
+ ClientConfig,
+ DependentEndpointConfig,
+ Endpoint,
+ EndpointResource,
ExternalWebhookInfo,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
+ WebhookSourceManager,
WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
- DependentEndpointConfig,
build_dependent_resource,
+ make_tracked_session,
+ rest_api_resource,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import BasePaginator
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- ClientConfig,
- Endpoint,
- EndpointResource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.workos.settings import (
- WAITLISTS_PARENT,
- WEBHOOK_SCHEMA_NAMES,
- WORKOS_ENDPOINTS,
-)
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.workos.settings import WAITLISTS_PARENT, WEBHOOK_SCHEMA_NAMES, WORKOS_ENDPOINTS
BASE_URL = "https://api.workos.com"
WEBHOOK_ENDPOINTS_PATH = "/webhook_endpoints"
diff --git a/products/warehouse_sources/sources/yousign/__init__.py b/products/warehouse_sources/sources/yousign/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/yousign.py b/products/warehouse_sources/sources/yousign/_config.py
similarity index 79%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/yousign.py
rename to products/warehouse_sources/sources/yousign/_config.py
index 09f2b28eee5f..9e71f994e7f5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/yousign.py
+++ b/products/warehouse_sources/sources/yousign/_config.py
@@ -3,7 +3,7 @@
from typing import Literal
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/canonical_descriptions.py b/products/warehouse_sources/sources/yousign/canonical_descriptions.py
similarity index 98%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/yousign/canonical_descriptions.py
rename to products/warehouse_sources/sources/yousign/canonical_descriptions.py
index 3c475331019b..d02a794cdfce 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/yousign/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"signature_requests": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/settings.py b/products/warehouse_sources/sources/yousign/settings.py
similarity index 95%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/yousign/settings.py
rename to products/warehouse_sources/sources/yousign/settings.py
index 941f59730ab8..2269115ac690 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/settings.py
+++ b/products/warehouse_sources/sources/yousign/settings.py
@@ -1,11 +1,7 @@
from dataclasses import dataclass, field
from typing import Any, Optional
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
- DependentEndpointConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
-from products.warehouse_sources.backend.types import IncrementalField
+from products.warehouse_sources.sources.sdk import DependentEndpointConfig, IncrementalField, incremental_field
PRODUCTION_BASE_URL = "https://api.yousign.app/v3"
SANDBOX_BASE_URL = "https://api-sandbox.yousign.app/v3"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/source.py b/products/warehouse_sources/sources/yousign/source.py
similarity index 84%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/yousign/source.py
rename to products/warehouse_sources/sources/yousign/source.py
index 4af3ec4866c9..8446c8bfd539 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/source.py
+++ b/products/warehouse_sources/sources/yousign/source.py
@@ -2,44 +2,38 @@ from typing import TYPE_CHECKING, Optional, cast
import structlog
-from products.warehouse_sources.backend.facade.codebase_detection import detect
+from products.warehouse_sources.sources.sdk import detect
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ ExternalWebhookInfo,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
SourceFieldSelectConfig,
SourceFieldSelectConfigOption,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- ExternalWebhookInfo,
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
+ WebhookSourceManager,
WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
build_endpoint_schemas,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.yousign import (
- YouSignSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.yousign.settings import (
+from products.warehouse_sources.sources.yousign._config import YouSignSourceConfig
+from products.warehouse_sources.sources.yousign.settings import (
ENDPOINT_DESCRIPTIONS,
ENDPOINTS,
INCREMENTAL_FIELDS,
@@ -47,7 +41,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.yousign.se
YOUSIGN_ENDPOINTS,
all_webhook_events,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.yousign.yousign import (
+from products.warehouse_sources.sources.yousign.yousign import (
YousignResumeConfig,
create_webhook as create_yousign_webhook,
delete_webhook as delete_yousign_webhook,
@@ -56,7 +50,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.yousign.yo
validate_credentials as validate_yousign_credentials,
yousign_source,
)
-from products.warehouse_sources.backend.types import ExternalDataSourceType
# Webhook management calls run in the API request path where no job logger exists.
logger = structlog.get_logger(__name__)
@@ -139,7 +132,7 @@ If automatic creation failed, note that webhook management requires a full-acces
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.yousign.canonical_descriptions import ( # noqa: PLC0415
+ from products.warehouse_sources.sources.yousign.canonical_descriptions import ( # noqa: PLC0415
CANONICAL_DESCRIPTIONS,
)
@@ -147,9 +140,7 @@ If automatic creation failed, note that webhook management requires a full-acces
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.yousign.webhook_template import ( # noqa: PLC0415
- template,
- )
+ from products.warehouse_sources.sources.yousign.webhook_template import template # noqa: PLC0415
return template
diff --git a/products/warehouse_sources/sources/yousign/tests/__init__.py b/products/warehouse_sources/sources/yousign/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/tests/test_yousign.py b/products/warehouse_sources/sources/yousign/tests/test_yousign.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/yousign/tests/test_yousign.py
rename to products/warehouse_sources/sources/yousign/tests/test_yousign.py
index 9a5dd249b477..b807c5cf6daa 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/tests/test_yousign.py
+++ b/products/warehouse_sources/sources/yousign/tests/test_yousign.py
@@ -7,24 +7,22 @@ from unittest import mock
import orjson
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import (
ScriptedResponse,
SourceDriver,
always,
route,
scripted_network,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.yousign import (
- YouSignSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.yousign.settings import (
+from products.warehouse_sources.sources.yousign._config import YouSignSourceConfig
+from products.warehouse_sources.sources.yousign.settings import (
SIGNATURE_REQUEST_SOURCES,
WEBHOOK_EVENTS,
YOUSIGN_ENDPOINTS,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.yousign.source import YouSignSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.yousign.yousign import (
+from products.warehouse_sources.sources.yousign.source import YouSignSource
+from products.warehouse_sources.sources.yousign.yousign import (
YousignResumeConfig,
_date_filter_value,
create_webhook,
@@ -160,7 +158,7 @@ class TestYousignSourceResponse:
# Patch out the real async_to_sync: running its event loop closes the thread-local Django
# connection, which poisons DB-backed tests (e.g. migration tests) later in the same worker.
@mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.yousign.yousign.async_to_sync",
+ "products.warehouse_sources.sources.yousign.yousign.async_to_sync",
lambda fn: fn,
)
def test_webhook_enabled_schema_reads_from_webhook_manager(self) -> None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/tests/test_yousign_source.py b/products/warehouse_sources/sources/yousign/tests/test_yousign_source.py
similarity index 81%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/yousign/tests/test_yousign_source.py
rename to products/warehouse_sources/sources/yousign/tests/test_yousign_source.py
index 10fa80791b7a..f4ce0346f27f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/tests/test_yousign_source.py
+++ b/products/warehouse_sources/sources/yousign/tests/test_yousign_source.py
@@ -1,10 +1,8 @@
from unittest import mock
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.yousign import (
- YouSignSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.yousign.settings import WEBHOOK_EVENTS
-from products.warehouse_sources.backend.temporal.data_imports.sources.yousign.source import YouSignSource
+from products.warehouse_sources.sources.yousign._config import YouSignSourceConfig
+from products.warehouse_sources.sources.yousign.settings import WEBHOOK_EVENTS
+from products.warehouse_sources.sources.yousign.source import YouSignSource
class TestYousignSource:
@@ -34,15 +32,13 @@ class TestYousignSource:
def test_desired_webhook_events_cover_all_mapped_events(self) -> None:
assert self.source.get_desired_webhook_events(self.config, ["signature_requests"]) == WEBHOOK_EVENTS
- @mock.patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.yousign.source.create_yousign_webhook"
- )
+ @mock.patch("products.warehouse_sources.sources.yousign.source.create_yousign_webhook")
def test_create_webhook_plumbs_config(self, mock_create: mock.MagicMock) -> None:
self.source.create_webhook(self.config, "https://ph/webhook", self.team_id)
args = mock_create.call_args.args
assert args[:3] == ("key", "production", "https://ph/webhook")
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.yousign.source.yousign_source")
+ @mock.patch("products.warehouse_sources.sources.yousign.source.yousign_source")
def test_source_for_pipeline_plumbs_arguments(self, mock_yousign_source: mock.MagicMock) -> None:
inputs = mock.MagicMock()
inputs.schema_name = "signature_requests"
@@ -63,7 +59,7 @@ class TestYousignSource:
assert kwargs["incremental_field"] == "completed_at"
assert kwargs["db_incremental_field_last_value"] == "2025-03-02"
- @mock.patch("products.warehouse_sources.backend.temporal.data_imports.sources.yousign.source.yousign_source")
+ @mock.patch("products.warehouse_sources.sources.yousign.source.yousign_source")
def test_source_for_pipeline_drops_watermark_for_full_refresh(self, mock_yousign_source: mock.MagicMock) -> None:
inputs = mock.MagicMock()
inputs.schema_name = "signature_requests"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/webhook_template.py b/products/warehouse_sources/sources/yousign/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/yousign/webhook_template.py
rename to products/warehouse_sources/sources/yousign/webhook_template.py
index c2526595c816..b9981f6ecf72 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/webhook_template.py
+++ b/products/warehouse_sources/sources/yousign/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
# Yousign signs deliveries with X-Yousign-Signature-256: sha256=<hex hmac-sha256 of the raw
# request body>, keyed with the subscription's secret_key (returned on webhook creation and
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/yousign.py b/products/warehouse_sources/sources/yousign/yousign.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/yousign/yousign.py
rename to products/warehouse_sources/sources/yousign/yousign.py
index acd7a92e692b..d3032a9c3fbb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/yousign/yousign.py
+++ b/products/warehouse_sources/sources/yousign/yousign.py
@@ -10,35 +10,27 @@ from asgiref.sync import async_to_sync
from dateutil import parser as dateutil_parser
from structlog.types import FilteringBoundLogger
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
+ ClientConfig,
+ Endpoint,
+ EndpointResource,
ExternalWebhookInfo,
+ JSONResponseCursorPaginator,
+ Resource,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SinglePagePaginator,
+ SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
+ WebhookSourceManager,
WebhookSyncResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
- rest_api_resource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.fanout import (
build_dependent_resource,
+ make_tracked_session,
+ rest_api_resource,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.paginators import (
- JSONResponseCursorPaginator,
- SinglePagePaginator,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- ClientConfig,
- Endpoint,
- EndpointResource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.yousign.settings import (
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.yousign.settings import (
PRODUCTION_BASE_URL,
SANDBOX_BASE_URL,
YOUSIGN_ENDPOINTS,
diff --git a/products/warehouse_sources/sources/zapsign/__init__.py b/products/warehouse_sources/sources/zapsign/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/zapsign.py b/products/warehouse_sources/sources/zapsign/_config.py
similarity index 79%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/zapsign.py
rename to products/warehouse_sources/sources/zapsign/_config.py
index 8d4925391d01..0751f58d27dc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/generated_configs/zapsign.py
+++ b/products/warehouse_sources/sources/zapsign/_config.py
@@ -3,7 +3,7 @@
from typing import Literal
-from products.warehouse_sources.backend.temporal.data_imports.sources.common import config
+from products.warehouse_sources.sources.sdk import config
@config.config
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/canonical_descriptions.py b/products/warehouse_sources/sources/zapsign/canonical_descriptions.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/canonical_descriptions.py
rename to products/warehouse_sources/sources/zapsign/canonical_descriptions.py
index 5ded652d25ff..9121b2b30cc2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/canonical_descriptions.py
+++ b/products/warehouse_sources/sources/zapsign/canonical_descriptions.py
@@ -1,6 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
+from products.warehouse_sources.sources.sdk import CanonicalDescriptions
CANONICAL_DESCRIPTIONS: CanonicalDescriptions = {
"documents": {
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/settings.py b/products/warehouse_sources/sources/zapsign/settings.py
similarity index 93%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/settings.py
rename to products/warehouse_sources/sources/zapsign/settings.py
index 914eb076e616..22e7df32c4a9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/settings.py
+++ b/products/warehouse_sources/sources/zapsign/settings.py
@@ -1,5 +1,4 @@
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import incremental_field
-from products.warehouse_sources.backend.types import IncrementalField, IncrementalFieldType
+from products.warehouse_sources.sources.sdk import IncrementalField, IncrementalFieldType, incremental_field
ZAPSIGN_BASE_URL = "https://api.zapsign.com.br"
ZAPSIGN_SANDBOX_BASE_URL = "https://sandbox.api.zapsign.com.br"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/source.py b/products/warehouse_sources/sources/zapsign/source.py
similarity index 81%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/source.py
rename to products/warehouse_sources/sources/zapsign/source.py
index ba29be38a339..cd9f6e5d9cd7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/source.py
+++ b/products/warehouse_sources/sources/zapsign/source.py
@@ -1,47 +1,36 @@
from typing import TYPE_CHECKING, Optional, cast
-from products.warehouse_sources.backend.facade.codebase_detection import detect
-from products.warehouse_sources.backend.facade.source_config import (
+from products.warehouse_sources.sources.sdk import (
+ CanonicalDescriptions,
DataWarehouseSourceCategory,
+ ExternalDataSourceType,
+ FieldType,
ReleaseStatus,
+ ResumableSource,
+ ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
SourceFieldSelectConfig,
SourceFieldSelectConfigOption,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
- FieldType,
- ResumableSource,
+ SourceInputs,
+ SourceRegistry,
+ SourceResponse,
+ SourceSchema,
WebhookCreationResult,
WebhookDeletionResult,
WebhookSource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.canonical_descriptions import (
- CanonicalDescriptions,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.registry import SourceRegistry
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.schema import (
- SourceSchema,
+ WebhookSourceManager,
build_endpoint_schemas,
+ detect,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.zapsign import (
- ZapSignSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign import zapsign as api_client
-from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.settings import (
- DOCUMENTS_RESOURCE,
- ENDPOINTS,
- INCREMENTAL_FIELDS,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.zapsign import ZapSignResumeConfig
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.sources.zapsign import zapsign as api_client
+from products.warehouse_sources.sources.zapsign._config import ZapSignSourceConfig
+from products.warehouse_sources.sources.zapsign.settings import DOCUMENTS_RESOURCE, ENDPOINTS, INCREMENTAL_FIELDS
+from products.warehouse_sources.sources.zapsign.zapsign import ZapSignResumeConfig
if TYPE_CHECKING:
- from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+ from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
@SourceRegistry.register
@@ -72,15 +61,13 @@ class ZapSignSource(
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
- from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.canonical_descriptions import (
- CANONICAL_DESCRIPTIONS,
- )
+ from products.warehouse_sources.sources.zapsign.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
@property
def webhook_template(self) -> Optional["HogFunctionTemplateDC"]:
- from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.webhook_template import template
+ from products.warehouse_sources.sources.zapsign.webhook_template import template
return template
diff --git a/products/warehouse_sources/sources/zapsign/tests/__init__.py b/products/warehouse_sources/sources/zapsign/tests/__init__.py
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/tests/test_zapsign.py b/products/warehouse_sources/sources/zapsign/tests/test_zapsign.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/tests/test_zapsign.py
rename to products/warehouse_sources/sources/zapsign/tests/test_zapsign.py
index 3df161fedb44..b77bcf1cd624 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/tests/test_zapsign.py
+++ b/products/warehouse_sources/sources/zapsign/tests/test_zapsign.py
@@ -7,17 +7,10 @@ import requests
from parameterized import parameterized
from requests import Response
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.testing import (
- ScriptedResponse,
- SourceDriver,
- always,
- scripted_network,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.zapsign import (
- ZapSignSourceConfig,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.settings import (
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.sdk.testing import ScriptedResponse, SourceDriver, always, scripted_network
+from products.warehouse_sources.sources.zapsign._config import ZapSignSourceConfig
+from products.warehouse_sources.sources.zapsign.settings import (
AUDIT_TRAIL_RESOURCE,
DOCUMENTS_RESOURCE,
SIGNER_VERIFICATIONS_RESOURCE,
@@ -27,8 +20,8 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.se
ZAPSIGN_BASE_URL,
ZAPSIGN_SANDBOX_BASE_URL,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.source import ZapSignSource
-from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.zapsign import (
+from products.warehouse_sources.sources.zapsign.source import ZapSignSource
+from products.warehouse_sources.sources.zapsign.zapsign import (
ZapSignResumeConfig,
_to_created_from,
_webhook_table_transformer,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/tests/test_zapsign_source.py b/products/warehouse_sources/sources/zapsign/tests/test_zapsign_source.py
similarity index 89%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/tests/test_zapsign_source.py
rename to products/warehouse_sources/sources/zapsign/tests/test_zapsign_source.py
index f2c2352cbaa5..26d0ab7a7e98 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/tests/test_zapsign_source.py
+++ b/products/warehouse_sources/sources/zapsign/tests/test_zapsign_source.py
@@ -5,11 +5,11 @@ from unittest import mock
from parameterized import parameterized
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.settings import DOCUMENTS_RESOURCE
-from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.source import ZapSignSource
+from products.warehouse_sources.sources.sdk import SourceInputs, SourceResponse
+from products.warehouse_sources.sources.zapsign.settings import DOCUMENTS_RESOURCE
+from products.warehouse_sources.sources.zapsign.source import ZapSignSource
-API_CLIENT_PATCH = "products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.zapsign"
+API_CLIENT_PATCH = "products.warehouse_sources.sources.zapsign.zapsign"
def _source_inputs(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/webhook_template.py b/products/warehouse_sources/sources/zapsign/webhook_template.py
similarity index 97%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/webhook_template.py
rename to products/warehouse_sources/sources/zapsign/webhook_template.py
index 196ef1935db4..1aa81cb56c31 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/webhook_template.py
+++ b/products/warehouse_sources/sources/zapsign/webhook_template.py
@@ -1,4 +1,4 @@
-from products.cdp.backend.templates.hog_function_template import HogFunctionTemplateDC
+from products.warehouse_sources.sources.sdk import HogFunctionTemplateDC
template: HogFunctionTemplateDC = HogFunctionTemplateDC(
status="alpha",
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py b/products/warehouse_sources/sources/zapsign/zapsign.py
similarity index 94%
rename from products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py
rename to products/warehouse_sources/sources/zapsign/zapsign.py
index b93fd76e54c1..58e92aecff5f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/zapsign/zapsign.py
+++ b/products/warehouse_sources/sources/zapsign/zapsign.py
@@ -19,25 +19,21 @@ from asgiref.sync import async_to_sync
from dateutil import parser as dateutil_parser
from requests.exceptions import HTTPError, RequestException
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import table_from_py_list
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (
+from products.warehouse_sources.sources.sdk import (
+ ClientConfig,
+ EndpointResource,
+ Resource,
+ RESTAPIConfig,
+ ResumableSourceManager,
+ SourceResponse,
WebhookCreationResult,
WebhookDeletionResult,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.http import make_tracked_session
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source import (
- RESTAPIConfig,
+ WebhookSourceManager,
+ make_tracked_session,
rest_api_resources,
)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.resource import Resource
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.rest_source.typing import (
- ClientConfig,
- EndpointResource,
-)
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.resumable import ResumableSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceResponse
-from products.warehouse_sources.backend.temporal.data_imports.sources.common.webhook_s3 import WebhookSourceManager
-from products.warehouse_sources.backend.temporal.data_imports.sources.zapsign.settings import (
+from products.warehouse_sources.sources.sdk.internals import table_from_py_list
+from products.warehouse_sources.sources.zapsign.settings import (
AUDIT_TRAIL_PATH,
AUDIT_TRAIL_RESOURCE,
DOCUMENT_DETAIL_PATH,
diff --git a/pyproject.toml b/pyproject.toml
index 74099ed1aadd..537ab3d71df5 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -628,6 +628,7 @@ ignore_imports = [
"products.warehouse_sources.sources.sdk -> products.**",
"products.warehouse_sources.sources.sdk.** -> posthog",
"products.warehouse_sources.sources.sdk.** -> products.**",
+ "products.warehouse_sources.sources.sdk.testing -> common",
"products.warehouse_sources.sources.conftest -> products.**",
]
diff --git a/tach.toml b/tach.toml
index 99f87036f682..bf4864c17bb4 100644
--- a/tach.toml
+++ b/tach.toml
@@ -34,10 +34,10 @@ depends_on = ["products.warehouse_sources.sources.sdk", "products.warehouse_sour
layer = "modules"
# The explicit entry wins over the glob for `products.warehouse_sources.sources.sdk`. The SDK re-exports the shared source code
-# of warehouse_sources, which still lives in the product.
+# of warehouse_sources, which still lives in the product, and the webhook template types of cdp.
[[modules]]
path = "products.warehouse_sources.sources.sdk"
-depends_on = ["posthog", "products.warehouse_sources"]
+depends_on = ["posthog", "products.warehouse_sources", "products.cdp"]
layer = "modules"
# Re-exports from product modules that warehouse_sources does not expose (pipeline core, naming