Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

8.6 KB
Raw Blame
from datetime import date, datetime
from typing import Any, cast

from requests import RequestException

from products.warehouse_sources.sources.sdk import (
    CanonicalDescriptions,
    DataWarehouseSourceCategory,
    ExternalDataSourceType,
    FieldType,
    ReleaseStatus,
    RESTClientRetryableError,
    ResumableSource,
    ResumableSourceManager,
    SourceConfig,
    SourceFieldInputConfig,
    SourceFieldInputConfigType,
    SourceInputs,
    SourceRegistry,
    SourceResponse,
    SourceSchema,
    detect,
)
from products.warehouse_sources.sources.serpstat._config import SerpstatSourceConfig
from products.warehouse_sources.sources.serpstat.canonical_descriptions import CANONICAL_DESCRIPTIONS
from products.warehouse_sources.sources.serpstat.serpstat import SerpstatResumeConfig, serpstat_resource
from products.warehouse_sources.sources.serpstat.settings import (
    AUTH_ERROR,
    DEFAULT_SEARCH_ENGINE,
    DOCS_BASE,
    DOMAIN_ERROR,
    ENDPOINTS,
    QUOTA_ERROR,
    REQUEST_ERROR,
)


@SourceRegistry.register
class SerpstatSource(ResumableSource[SerpstatSourceConfig, SerpstatResumeConfig]):
    codebase_detection = detect(env=("^SERPSTAT_",))

    lists_tables_without_credentials = True
    supported_versions = ("v4",)
    default_version = "v4"
    api_docs_url = f"{DOCS_BASE}jenasqbwtxdlr-introduction-to-serpstat-api"

    @property
    def source_type(self) -> ExternalDataSourceType:
        return ExternalDataSourceType.SERPSTAT

    def get_non_retryable_errors(self) -> dict[str, str | None]:
        return {message: message for message in (AUTH_ERROR, QUOTA_ERROR, REQUEST_ERROR, DOMAIN_ERROR)}

    def get_canonical_descriptions(self) -> CanonicalDescriptions:
        return CANONICAL_DESCRIPTIONS

    def get_schemas(
        self,
        config: SerpstatSourceConfig,
        team_id: int,
        with_counts: bool = False,
        names: list[str] | None = None,
        force_refresh: bool = False,
        api_version: str | None = None,
    ) -> list[SourceSchema]:
        return [
            SourceSchema(
                name=name,
                supports_incremental=bool(endpoint.incremental_fields),
                supports_append=False,
                incremental_fields=endpoint.incremental_fields,
                detected_primary_keys=list(endpoint.primary_keys),
                description=CANONICAL_DESCRIPTIONS[name]["description"],
            )
            for name, endpoint in ENDPOINTS.items()
            if names is None or name in names
        ]

    def validate_credentials(
        self,
        config: SerpstatSourceConfig,
        team_id: int,
        schema_name: str | None = None,
        api_version: str | None = None,
    ) -> tuple[bool, str | None]:
        for label, value in (("project ID", config.project_id), ("project region ID", config.project_region_id)):
            try:
                if int(value) <= 0:
                    raise ValueError
            except (ValueError, TypeError):
                return False, f"Enter a positive integer for the Serpstat {label}."
        try:
            list(
                serpstat_resource(
                    config, "projects", team_id, "", self.resolve_api_version(api_version), credential_check=True
                )
            )
        except ValueError as error:
            message = str(error)
            return False, message if message in self.get_non_retryable_errors() else REQUEST_ERROR
        except (RequestException, RESTClientRetryableError):
            return False, "Could not reach Serpstat. Try again later."
        return True, None

    def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[SerpstatResumeConfig]:
        return ResumableSourceManager(inputs, SerpstatResumeConfig)

    def source_for_pipeline(
        self,
        config: SerpstatSourceConfig,
        resumable_source_manager: ResumableSourceManager[SerpstatResumeConfig],
        inputs: SourceInputs,
    ) -> SourceResponse:
        endpoint = ENDPOINTS[inputs.schema_name]
        resource = serpstat_resource(
            config,
            inputs.schema_name,
            inputs.team_id,
            inputs.job_id,
            self.resolve_api_version(inputs.api_version),
            resumable_source_manager,
            date_from=_date_from(inputs.db_incremental_field_last_value)
            if inputs.should_use_incremental_field
            else None,
        )
        return SourceResponse(
            name=resource.name,
            items=lambda: resource,
            primary_keys=list(endpoint.primary_keys),
            partition_keys=[endpoint.partition_key] if endpoint.partition_key else None,
            partition_mode="datetime" if endpoint.partition_key else None,
            partition_format="month" if endpoint.partition_key else None,
            sort_mode="asc" if inputs.schema_name in ("project_keywords", "project_positions") else None,
        )

    @property
    def get_source_config(self) -> SourceConfig:
        return SourceConfig(
            name=ExternalDataSourceType.SERPSTAT,
            category=DataWarehouseSourceCategory.ANALYTICS,
            label="Serpstat",
            iconPath="/static/services/serpstat.png",
            docsUrl="https://posthog.com/docs/cdp/sources/serpstat",
            releaseStatus=ReleaseStatus.BETA,
            caption=(
                "Create an API token on your [Serpstat profile](https://serpstat.com/users/profile/). "
                "API access requires the Team plan or higher. "
                "Syncs use your account's API units when Serpstat applies a charge. "
                "Serpstat lists the project methods as free of API credit charges. "
                "The domain and backlink tables use API credits for each returned row, and they need a domain. "
                "Paginated tables include up to 500 records. Positions cover the last seven days for one project region."
            ),
            fields=cast(
                list[FieldType],
                [
                    SourceFieldInputConfig(
                        name="api_key",
                        label="API token",
                        type=SourceFieldInputConfigType.PASSWORD,
                        required=True,
                        placeholder="",
                        secret=True,
                    ),
                    SourceFieldInputConfig(
                        name="project_id",
                        label="Project ID",
                        type=SourceFieldInputConfigType.TEXT,
                        required=True,
                        placeholder="123456",
                        secret=False,
                        caption="Copy the project ID from the URL of your Serpstat Rank Tracker report.",
                    ),
                    SourceFieldInputConfig(
                        name="project_region_id",
                        label="Project region ID",
                        type=SourceFieldInputConfigType.TEXT,
                        required=True,
                        placeholder="123456",
                        secret=False,
                        caption="Use the region ID from the Serpstat getProjectRegions method.",
                    ),
                    SourceFieldInputConfig(
                        name="domain",
                        label="Domain",
                        type=SourceFieldInputConfigType.TEXT,
                        required=False,
                        placeholder="example.com",
                        secret=False,
                        caption="The domain to analyze for the domain and backlink tables.",
                    ),
                    SourceFieldInputConfig(
                        name="search_engine",
                        sync_setting=True,
                        label="Search engine database",
                        type=SourceFieldInputConfigType.TEXT,
                        required=False,
                        placeholder=DEFAULT_SEARCH_ENGINE,
                        secret=False,
                        caption=(
                            "The Serpstat database code for the domain tables, for example `g_us` for Google US. "
                            f"Leave blank to use `{DEFAULT_SEARCH_ENGINE}`. See the "
                            f"[database list]({DOCS_BASE}ba97ni814ao9p-search-engine-short-names)."
                        ),
                    ),
                ],
            ),
        )


def _date_from(value: Any) -> str | None:
    if isinstance(value, datetime):
        return value.date().isoformat()
    if isinstance(value, date):
        return value.isoformat()
    if isinstance(value, str) and value:
        return value[:10]
    return None