:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from datetime import date, datetime
from typing import Any, cast
from requests import RequestException
from products.warehouse_sources.sources.sdk import (
CanonicalDescriptions,
DataWarehouseSourceCategory,
ExternalDataSourceType,
FieldType,
ReleaseStatus,
RESTClientRetryableError,
ResumableSource,
ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
SourceInputs,
SourceRegistry,
SourceResponse,
SourceSchema,
detect,
)
from products.warehouse_sources.sources.serpstat._config import SerpstatSourceConfig
from products.warehouse_sources.sources.serpstat.canonical_descriptions import CANONICAL_DESCRIPTIONS
from products.warehouse_sources.sources.serpstat.serpstat import SerpstatResumeConfig, serpstat_resource
from products.warehouse_sources.sources.serpstat.settings import (
AUTH_ERROR,
DEFAULT_SEARCH_ENGINE,
DOCS_BASE,
DOMAIN_ERROR,
ENDPOINTS,
QUOTA_ERROR,
REQUEST_ERROR,
)
@SourceRegistry.register
class SerpstatSource(ResumableSource[SerpstatSourceConfig, SerpstatResumeConfig]):
codebase_detection = detect(env=("^SERPSTAT_",))
lists_tables_without_credentials = True
supported_versions = ("v4",)
default_version = "v4"
api_docs_url = f"{DOCS_BASE}jenasqbwtxdlr-introduction-to-serpstat-api"
@property
def source_type(self) -> ExternalDataSourceType:
return ExternalDataSourceType.SERPSTAT
def get_non_retryable_errors(self) -> dict[str, str | None]:
return {message: message for message in (AUTH_ERROR, QUOTA_ERROR, REQUEST_ERROR, DOMAIN_ERROR)}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
return CANONICAL_DESCRIPTIONS
def get_schemas(
self,
config: SerpstatSourceConfig,
team_id: int,
with_counts: bool = False,
names: list[str] | None = None,
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
return [
SourceSchema(
name=name,
supports_incremental=bool(endpoint.incremental_fields),
supports_append=False,
incremental_fields=endpoint.incremental_fields,
detected_primary_keys=list(endpoint.primary_keys),
description=CANONICAL_DESCRIPTIONS[name]["description"],
)
for name, endpoint in ENDPOINTS.items()
if names is None or name in names
]
def validate_credentials(
self,
config: SerpstatSourceConfig,
team_id: int,
schema_name: str | None = None,
api_version: str | None = None,
) -> tuple[bool, str | None]:
for label, value in (("project ID", config.project_id), ("project region ID", config.project_region_id)):
try:
if int(value) <= 0:
raise ValueError
except (ValueError, TypeError):
return False, f"Enter a positive integer for the Serpstat {label}."
try:
list(
serpstat_resource(
config, "projects", team_id, "", self.resolve_api_version(api_version), credential_check=True
)
)
except ValueError as error:
message = str(error)
return False, message if message in self.get_non_retryable_errors() else REQUEST_ERROR
except (RequestException, RESTClientRetryableError):
return False, "Could not reach Serpstat. Try again later."
return True, None
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[SerpstatResumeConfig]:
return ResumableSourceManager(inputs, SerpstatResumeConfig)
def source_for_pipeline(
self,
config: SerpstatSourceConfig,
resumable_source_manager: ResumableSourceManager[SerpstatResumeConfig],
inputs: SourceInputs,
) -> SourceResponse:
endpoint = ENDPOINTS[inputs.schema_name]
resource = serpstat_resource(
config,
inputs.schema_name,
inputs.team_id,
inputs.job_id,
self.resolve_api_version(inputs.api_version),
resumable_source_manager,
date_from=_date_from(inputs.db_incremental_field_last_value)
if inputs.should_use_incremental_field
else None,
)
return SourceResponse(
name=resource.name,
items=lambda: resource,
primary_keys=list(endpoint.primary_keys),
partition_keys=[endpoint.partition_key] if endpoint.partition_key else None,
partition_mode="datetime" if endpoint.partition_key else None,
partition_format="month" if endpoint.partition_key else None,
sort_mode="asc" if inputs.schema_name in ("project_keywords", "project_positions") else None,
)
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
name=ExternalDataSourceType.SERPSTAT,
category=DataWarehouseSourceCategory.ANALYTICS,
label="Serpstat",
iconPath="/static/services/serpstat.png",
docsUrl="https://posthog.com/docs/cdp/sources/serpstat",
releaseStatus=ReleaseStatus.BETA,
caption=(
"Create an API token on your [Serpstat profile](https://serpstat.com/users/profile/). "
"API access requires the Team plan or higher. "
"Syncs use your account's API units when Serpstat applies a charge. "
"Serpstat lists the project methods as free of API credit charges. "
"The domain and backlink tables use API credits for each returned row, and they need a domain. "
"Paginated tables include up to 500 records. Positions cover the last seven days for one project region."
),
fields=cast(
list[FieldType],
[
SourceFieldInputConfig(
name="api_key",
label="API token",
type=SourceFieldInputConfigType.PASSWORD,
required=True,
placeholder="",
secret=True,
),
SourceFieldInputConfig(
name="project_id",
label="Project ID",
type=SourceFieldInputConfigType.TEXT,
required=True,
placeholder="123456",
secret=False,
caption="Copy the project ID from the URL of your Serpstat Rank Tracker report.",
),
SourceFieldInputConfig(
name="project_region_id",
label="Project region ID",
type=SourceFieldInputConfigType.TEXT,
required=True,
placeholder="123456",
secret=False,
caption="Use the region ID from the Serpstat getProjectRegions method.",
),
SourceFieldInputConfig(
name="domain",
label="Domain",
type=SourceFieldInputConfigType.TEXT,
required=False,
placeholder="example.com",
secret=False,
caption="The domain to analyze for the domain and backlink tables.",
),
SourceFieldInputConfig(
name="search_engine",
sync_setting=True,
label="Search engine database",
type=SourceFieldInputConfigType.TEXT,
required=False,
placeholder=DEFAULT_SEARCH_ENGINE,
secret=False,
caption=(
"The Serpstat database code for the domain tables, for example `g_us` for Google US. "
f"Leave blank to use `{DEFAULT_SEARCH_ENGINE}`. See the "
f"[database list]({DOCS_BASE}ba97ni814ao9p-search-engine-short-names)."
),
),
],
),
)
def _date_from(value: Any) -> str | None:
if isinstance(value, datetime):
return value.date().isoformat()
if isinstance(value, date):
return value.isoformat()
if isinstance(value, str) and value:
return value[:10]
return None