Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

14.8 KB
Raw Blame
from typing import Optional, cast

from products.warehouse_sources.sources.commercetools._config import CommercetoolsSourceConfig
from products.warehouse_sources.sources.commercetools.commercetools import (
    CommercetoolsResumeConfig,
    commercetools_source,
    validate_credentials as validate_commercetools_credentials,
)
from products.warehouse_sources.sources.commercetools.settings import ENDPOINTS, INCREMENTAL_FIELDS, SHOULD_SYNC_DEFAULT
from products.warehouse_sources.sources.sdk import (
    CanonicalDescriptions,
    DataWarehouseSourceCategory,
    ExternalDataSourceType,
    FieldType,
    ReleaseStatus,
    ResumableSource,
    ResumableSourceManager,
    SourceConfig,
    SourceFieldInputConfig,
    SourceFieldInputConfigType,
    SourceFieldSelectConfig,
    SourceFieldSelectConfigOption,
    SourceInputs,
    SourceRegistry,
    SourceResponse,
    SourceSchema,
    build_endpoint_schemas,
    detect,
)


@SourceRegistry.register
class CommercetoolsSource(ResumableSource[CommercetoolsSourceConfig, CommercetoolsResumeConfig]):
    codebase_detection = detect(
        env=("^CTP_", "^COMMERCETOOLS_"),
        npm=(
            "@commercetools/platform-sdk",
            "@commercetools/ts-client",
            "@commercetools/sdk-client-v2",
            "@commercetools/importapi-sdk",
        ),
        pypi=("commercetools",),
        composer=(
            "commercetools/commercetools-sdk",
            "commercetools/php-sdk",
            "commercetools/commercetools-api-reference",
        ),
        go=("github.com/labd/commercetools-go-sdk",),
        maven=(
            "com.commercetools.sdk:commercetools-sdk-java-api",
            "com.commercetools.sdk:commercetools-apachehttp-client",
            "com.commercetools.sdk:commercetools-async-http-client",
            "com.commercetools.sdk:commercetools-graphql-api",
            "com.commercetools.sdk:commercetools-http-client",
            "com.commercetools.sdk:commercetools-importapi-utils",
            "com.commercetools.sdk:commercetools-javanet-client",
            "com.commercetools.sdk:commercetools-money",
            "com.commercetools.sdk:commercetools-monitoring-datadog",
            "com.commercetools.sdk:commercetools-monitoring-newrelic",
            "com.commercetools.sdk:commercetools-monitoring-opentelemetry",
            "com.commercetools.sdk:commercetools-okhttp-client3",
            "com.commercetools.sdk:commercetools-okhttp-client4",
            "com.commercetools.sdk:commercetools-okhttp-client5",
            "com.commercetools.sdk:commercetools-reactornetty-client",
            "com.commercetools.sdk:commercetools-sdk-compat-v1",
            "com.commercetools.sdk:commercetools-sdk-java-api-base",
            "com.commercetools.sdk:commercetools-sdk-java-api-models-base",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_agent",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_api_client",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_approval_flow",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_approval_rule",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_associate_role",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_attribute_group",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_business_unit",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_business_unit_search",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_cart",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_cart_discount",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_category",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_channel",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_common",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_custom_object",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_customer",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_customer_group",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_customer_search",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_discount_code",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_discount_group",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_error",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_event",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_extension",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_graph_ql",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_inventory",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_mcp_server",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_me",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_message",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_order",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_order_edit",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_payment",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_payment_method",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_product",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_product_discount",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_product_search",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_product_selection",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_product_tailoring",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_product_type",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_project",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_quote",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_quote_request",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_recurrence_policy",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_recurring_order",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_reservation",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_review",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_search",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_shipping_method",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_shopping_list",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_staged_quote",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_standalone_price",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_state",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_store",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_store_country",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_subscription",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_tax_category",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_type",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_variant",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_variant_attributes",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_warning",
            "com.commercetools.sdk:commercetools-sdk-java-api-models_zone",
            "com.commercetools.sdk:commercetools-sdk-java-api-predicates",
            "com.commercetools.sdk:commercetools-sdk-java-checkout",
            "com.commercetools.sdk:commercetools-sdk-java-history",
            "com.commercetools.sdk:commercetools-sdk-java-importapi",
            "com.commercetools.sdk:commercetools-sdk-java-ml",
        ),
    )

    lists_tables_without_credentials = True  # static endpoint catalog — safe for public docs

    api_docs_url = "https://docs.commercetools.com/api/"

    @property
    def source_type(self) -> ExternalDataSourceType:
        return ExternalDataSourceType.COMMERCETOOLS

    @property
    def connection_host_fields(self) -> list[str]:
        # `region` picks the host and `project_key` the project the stored client secret (and
        # minted bearer token) are sent to; retargeting either must re-require the secret so it
        # can't be aimed at a different commercetools project or region.
        return ["region", "project_key"]

    def get_canonical_descriptions(self) -> CanonicalDescriptions:
        from products.warehouse_sources.sources.commercetools.canonical_descriptions import CANONICAL_DESCRIPTIONS

        return CANONICAL_DESCRIPTIONS

    def get_non_retryable_errors(self) -> dict[str, str | None]:
        return {
            "401 Client Error: Unauthorized for url: https://auth.": "commercetools authentication failed. Please check your client ID and client secret.",
            "400 Client Error: Bad Request for url: https://auth.": "commercetools authentication failed. Please check your client ID and client secret.",
            "403 Client Error: Forbidden for url: https://api.": "commercetools denied access. Please check that your API client has the required view scope for this dataset.",
            "404 Client Error: Not Found for url: https://api.": "commercetools project not found. Please check your project key and region.",
        }

    @property
    def get_source_config(self) -> SourceConfig:
        return SourceConfig(
            name=ExternalDataSourceType.COMMERCETOOLS,
            category=DataWarehouseSourceCategory.E_COMMERCE,
            label="commercetools",
            caption="""Enter your commercetools API client credentials to pull your commerce data into the PostHog Data warehouse.

Create an API client in the Merchant Center under Settings > Developer settings with the view scopes for the datasets you want to sync (`view_orders`, `view_customers`, `view_payments`, `view_shopping_lists`, `view_products`, `view_product_types`, `view_categories`, `view_discount_codes`, `view_standalone_prices`, `view_stores`, `view_channels`, `view_customer_groups`, `view_states`, `view_messages`). Your project key and region are shown alongside the generated credentials.

The messages table also needs the Messages Query feature turned on in your project settings. Without it commercetools keeps no messages and the table syncs empty. commercetools also deletes messages once they pass your project's retention period, so the table covers that window rather than all history.""",
            iconPath="/static/services/commercetools.png",
            docsUrl="https://posthog.com/docs/cdp/sources/commercetools",
            releaseStatus=ReleaseStatus.BETA,
            fields=cast(
                list[FieldType],
                [
                    SourceFieldSelectConfig(
                        name="region",
                        label="Region",
                        required=True,
                        defaultValue="us-central1.gcp",
                        options=[
                            SourceFieldSelectConfigOption(
                                label="North America (GCP, us-central1)", value="us-central1.gcp"
                            ),
                            SourceFieldSelectConfigOption(
                                label="North America (AWS, us-east-2)", value="us-east-2.aws"
                            ),
                            SourceFieldSelectConfigOption(label="Europe (GCP, europe-west1)", value="europe-west1.gcp"),
                            SourceFieldSelectConfigOption(label="Europe (AWS, eu-central-1)", value="eu-central-1.aws"),
                            SourceFieldSelectConfigOption(
                                label="Australia (GCP, australia-southeast1)", value="australia-southeast1.gcp"
                            ),
                        ],
                    ),
                    SourceFieldInputConfig(
                        name="project_key",
                        label="Project key",
                        type=SourceFieldInputConfigType.TEXT,
                        required=True,
                        placeholder="my-project",
                        secret=False,
                    ),
                    SourceFieldInputConfig(
                        name="client_id",
                        label="Client ID",
                        type=SourceFieldInputConfigType.TEXT,
                        required=True,
                        placeholder="",
                        secret=False,
                    ),
                    SourceFieldInputConfig(
                        name="client_secret",
                        label="Client secret",
                        type=SourceFieldInputConfigType.PASSWORD,
                        required=True,
                        placeholder="",
                        secret=True,
                    ),
                ],
            ),
        )

    def get_schemas(
        self,
        config: CommercetoolsSourceConfig,
        team_id: int,
        with_counts: bool = False,
        names: list[str] | None = None,
        force_refresh: bool = False,
        api_version: str | None = None,
    ) -> list[SourceSchema]:
        return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, should_sync_default=SHOULD_SYNC_DEFAULT)

    def validate_credentials(
        self,
        config: CommercetoolsSourceConfig,
        team_id: int,
        schema_name: Optional[str] = None,
        api_version: str | None = None,
    ) -> tuple[bool, str | None]:
        if validate_commercetools_credentials(
            config.region, config.project_key, config.client_id, config.client_secret
        ):
            return True, None

        return False, "Invalid commercetools API client credentials"

    def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[CommercetoolsResumeConfig]:
        return ResumableSourceManager[CommercetoolsResumeConfig](inputs, CommercetoolsResumeConfig)

    def source_for_pipeline(
        self,
        config: CommercetoolsSourceConfig,
        resumable_source_manager: ResumableSourceManager[CommercetoolsResumeConfig],
        inputs: SourceInputs,
    ) -> SourceResponse:
        return commercetools_source(
            region=config.region,
            project_key=config.project_key,
            client_id=config.client_id,
            client_secret=config.client_secret,
            endpoint=inputs.schema_name,
            logger=inputs.logger,
            resumable_source_manager=resumable_source_manager,
            should_use_incremental_field=inputs.should_use_incremental_field,
            db_incremental_field_last_value=inputs.db_incremental_field_last_value
            if inputs.should_use_incremental_field
            else None,
        )