Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.9 KB
Raw Blame
from typing import Optional, cast

from products.warehouse_sources.sources.close._config import CloseSourceConfig
from products.warehouse_sources.sources.close.close import (
    CLOSE_BASE_URL,
    CloseResumeConfig,
    close_source,
    validate_credentials as validate_close_credentials,
)
from products.warehouse_sources.sources.close.settings import CLOSE_ENDPOINTS
from products.warehouse_sources.sources.sdk import (
    CanonicalDescriptions,
    DataWarehouseSourceCategory,
    ExternalDataSourceType,
    FieldType,
    ReleaseStatus,
    ResumableSource,
    ResumableSourceManager,
    SourceConfig,
    SourceFieldInputConfig,
    SourceFieldInputConfigType,
    SourceInputs,
    SourceRegistry,
    SourceResponse,
    SourceSchema,
    detect,
)


@SourceRegistry.register
class CloseSource(ResumableSource[CloseSourceConfig, CloseResumeConfig]):
    codebase_detection = detect(env=("^CLOSE_API_KEY$",), pypi=("closeio",))

    lists_tables_without_credentials = True  # static endpoint catalog — safe for public docs

    supported_versions = ("v1",)
    default_version = "v1"
    api_docs_url = "https://developer.close.com/"

    @property
    def source_type(self) -> ExternalDataSourceType:
        return ExternalDataSourceType.CLOSE

    def get_non_retryable_errors(self) -> dict[str, str | None]:
        return {
            "401 Client Error: Unauthorized for url": "Close authentication failed. Please check your API key.",
            "403 Client Error: Forbidden for url": "Your Close API key does not have access to this resource. Please check the key's permissions.",
            # Close rejects an over-long `_fields` list this way. The search now asks for custom
            # fields with a single selector, so this should not recur, but a stuck sync must stop
            # rather than let Temporal retry a request that can never succeed.
            "List is too long.": "Close rejected the request because the field list was too long. This can happen on an account with a very large number of custom fields. Contact PostHog support so we can adjust how the Close source requests them.",
        }

    def get_retryable_errors(self) -> set[str]:
        # Both `CLOSE_RETRY` (Advanced Filtering search) and `DEFAULT_RETRY` (the list endpoints)
        # already retry a dropped connection or read timeout before re-raising once that budget is
        # exhausted. urllib3 wraps all of those as "... Max retries exceeded with url: ..."
        # regardless of the underlying cause (refused connection, read timeout, dropped socket), so
        # match that stable prefix rather than the per-request URL or nested error detail. Temporal
        # then retries the whole activity, so the failure is transient and self-recovering.
        return {
            "Max retries exceeded with url",
            # `close_organizations_source` calls `/me/` and `/organization/{id}/` directly with
            # `raise_for_status()` rather than through the shared REST engine, so a persistent 5xx
            # there isn't wrapped as `RESTClientRetryableError`. `DEFAULT_RETRY` already retried the
            # request in-process before this reaches us, so a Close-side 500 here is a transient
            # upstream blip, not a bug. Match the stable prefix (status + reason + endpoint), not
            # the organization id in the URL.
            f"500 Server Error: Internal Server Error for url: {CLOSE_BASE_URL}/organization/",
        }

    def get_canonical_descriptions(self) -> CanonicalDescriptions:
        from products.warehouse_sources.sources.close.canonical_descriptions import CANONICAL_DESCRIPTIONS

        return CANONICAL_DESCRIPTIONS

    def get_schemas(
        self,
        config: CloseSourceConfig,
        team_id: int,
        with_counts: bool = False,
        names: list[str] | None = None,
        force_refresh: bool = False,
        api_version: str | None = None,
    ) -> list[SourceSchema]:
        schemas = [
            SourceSchema(
                name=endpoint_config.name,
                supports_incremental=len(endpoint_config.incremental_fields) > 0,
                supports_append=len(endpoint_config.incremental_fields) > 0,
                incremental_fields=endpoint_config.incremental_fields,
            )
            for endpoint_config in CLOSE_ENDPOINTS.values()
        ]

        if names is not None:
            names_set = set(names)
            schemas = [s for s in schemas if s.name in names_set]

        return schemas

    def validate_credentials(
        self, config: CloseSourceConfig, team_id: int, schema_name: Optional[str] = None, api_version: str | None = None
    ) -> tuple[bool, str | None]:
        if not config.api_key:
            return False, "Close API key is required"

        if validate_close_credentials(config.api_key):
            return True, None

        return False, "Invalid Close API key"

    def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[CloseResumeConfig]:
        return ResumableSourceManager[CloseResumeConfig](inputs, CloseResumeConfig)

    def source_for_pipeline(
        self,
        config: CloseSourceConfig,
        resumable_source_manager: ResumableSourceManager[CloseResumeConfig],
        inputs: SourceInputs,
    ) -> SourceResponse:
        return close_source(
            api_key=config.api_key,
            endpoint=inputs.schema_name,
            team_id=inputs.team_id,
            job_id=inputs.job_id,
            resumable_source_manager=resumable_source_manager,
            logger=inputs.logger,
            should_use_incremental_field=inputs.should_use_incremental_field,
            incremental_field=inputs.incremental_field,
            db_incremental_field_last_value=inputs.db_incremental_field_last_value
            if inputs.should_use_incremental_field
            else None,
        )

    @property
    def get_source_config(self) -> SourceConfig:
        return SourceConfig(
            name=ExternalDataSourceType.CLOSE,
            category=DataWarehouseSourceCategory.CRM,
            label="Close",
            caption=(
                "Import your CRM data from Close. Create an API key under "
                "**Settings → Developer → API Keys** in Close and paste it below. "
                "The key needs read access to the resources you want to sync."
            ),
            iconPath="/static/services/close.png",
            docsUrl="https://posthog.com/docs/cdp/sources/close",
            releaseStatus=ReleaseStatus.GA,
            fields=cast(
                list[FieldType],
                [
                    SourceFieldInputConfig(
                        name="api_key",
                        label="API key",
                        type=SourceFieldInputConfigType.PASSWORD,
                        required=True,
                        placeholder="api_...",
                        secret=True,
                    ),
                ],
            ),
        )