Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.8 KB
Raw Blame
from typing import Optional, cast

from products.warehouse_sources.sources.phyllo._config import PhylloSourceConfig
from products.warehouse_sources.sources.phyllo.phyllo import (
    PhylloResumeConfig,
    endpoint_supports_resume,
    phyllo_source,
    validate_credentials,
)
from products.warehouse_sources.sources.phyllo.settings import ENDPOINTS, INCREMENTAL_FIELDS, PHYLLO_ENDPOINTS
from products.warehouse_sources.sources.sdk import (
    CanonicalDescriptions,
    DataWarehouseSourceCategory,
    ExternalDataSourceType,
    FieldType,
    ReleaseStatus,
    ResumableSource,
    ResumableSourceManager,
    SourceConfig,
    SourceFieldInputConfig,
    SourceFieldInputConfigType,
    SourceFieldSelectConfig,
    SourceFieldSelectConfigOption,
    SourceInputs,
    SourceRegistry,
    SourceResponse,
    SourceSchema,
    build_endpoint_schemas,
    detect,
)


@SourceRegistry.register
class PhylloSource(ResumableSource[PhylloSourceConfig, PhylloResumeConfig]):
    codebase_detection = detect(
        env=("^PHYLLO_",),
        npm=("react-native-phyllo-connect", "phyllo-connect"),
        cocoapods=("PhylloConnect",),
        pub=("phyllo_connect",),
    )

    lists_tables_without_credentials = True  # static endpoint catalog — safe for public docs
    supported_versions = ("v1",)
    default_version = "v1"
    api_docs_url = "https://docs.getphyllo.com/docs/api-reference/introduction/introduction"

    @property
    def source_type(self) -> ExternalDataSourceType:
        return ExternalDataSourceType.PHYLLO

    @property
    def get_source_config(self) -> SourceConfig:
        return SourceConfig(
            name=ExternalDataSourceType.PHYLLO,
            category=DataWarehouseSourceCategory.ANALYTICS,
            label="Phyllo",
            releaseStatus=ReleaseStatus.BETA,
            caption="""Enter your Phyllo API credentials to pull creator accounts, profiles, content, and income data into the PostHog Data warehouse.

You can find your client ID and secret in the [Phyllo developer dashboard](https://dashboard.getphyllo.com) under **API credentials**. Credentials are environment-specific — sandbox credentials only authenticate against the sandbox environment.
""",
            iconPath="/static/services/phyllo.png",
            docsUrl="https://posthog.com/docs/cdp/sources/phyllo",
            keywords=["creator", "getphyllo"],
            fields=cast(
                list[FieldType],
                [
                    SourceFieldInputConfig(
                        name="client_id",
                        label="Client ID",
                        type=SourceFieldInputConfigType.TEXT,
                        required=True,
                        placeholder="",
                        secret=False,
                    ),
                    SourceFieldInputConfig(
                        name="client_secret",
                        label="Client secret",
                        type=SourceFieldInputConfigType.PASSWORD,
                        required=True,
                        placeholder="",
                        secret=True,
                    ),
                    SourceFieldSelectConfig(
                        name="environment",
                        label="Environment",
                        required=True,
                        defaultValue="production",
                        options=[
                            SourceFieldSelectConfigOption(label="Production", value="production"),
                            SourceFieldSelectConfigOption(label="Sandbox", value="sandbox"),
                        ],
                    ),
                ],
            ),
        )

    def get_canonical_descriptions(self) -> CanonicalDescriptions:
        from products.warehouse_sources.sources.phyllo.canonical_descriptions import (
            CANONICAL_DESCRIPTIONS,  # noqa: PLC0415
        )

        return CANONICAL_DESCRIPTIONS

    def get_non_retryable_errors(self) -> dict[str, str | None]:
        message = "Your Phyllo client ID or secret is invalid for the selected environment. Check the credentials in the Phyllo developer dashboard, then reconnect."
        return {
            "401 Client Error: Unauthorized for url: https://api.getphyllo.com": message,
            "401 Client Error: Unauthorized for url: https://api.sandbox.getphyllo.com": message,
            "403 Client Error: Forbidden for url: https://api.getphyllo.com": message,
            "403 Client Error: Forbidden for url: https://api.sandbox.getphyllo.com": message,
        }

    def get_schemas(
        self,
        config: PhylloSourceConfig,
        team_id: int,
        with_counts: bool = False,
        names: list[str] | None = None,
        force_refresh: bool = False,
        api_version: str | None = None,
    ) -> list[SourceSchema]:
        # Every endpoint is full refresh only (INCREMENTAL_FIELDS is empty) — see the note in
        # settings.py on the unverified from_date/to_date filters.
        return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)

    def validate_credentials(
        self,
        config: PhylloSourceConfig,
        team_id: int,
        schema_name: Optional[str] = None,
        api_version: str | None = None,
    ) -> tuple[bool, str | None]:
        # Credentials are environment-wide, so a single probe validates access to every schema.
        return validate_credentials(config.client_id, config.client_secret, config.environment)

    def resume_covers_run(
        self,
        *,
        incremental_or_append: bool,
        schema_name: str | None = None,
    ) -> bool:
        # An endpoint that cannot checkpoint restarts from the first account on every attempt, so the
        # larger resumable retry budget would only repeat the whole fan-out.
        return schema_name is None or schema_name not in PHYLLO_ENDPOINTS or endpoint_supports_resume(schema_name)

    def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[PhylloResumeConfig]:
        return ResumableSourceManager[PhylloResumeConfig](inputs, PhylloResumeConfig)

    def source_for_pipeline(
        self,
        config: PhylloSourceConfig,
        resumable_source_manager: ResumableSourceManager[PhylloResumeConfig],
        inputs: SourceInputs,
    ) -> SourceResponse:
        if inputs.schema_name not in PHYLLO_ENDPOINTS:
            raise ValueError(f"Unknown Phyllo schema '{inputs.schema_name}'")

        return phyllo_source(
            client_id=config.client_id,
            client_secret=config.client_secret,
            environment=config.environment,
            endpoint=inputs.schema_name,
            team_id=inputs.team_id,
            job_id=inputs.job_id,
            resumable_source_manager=resumable_source_manager,
            db_incremental_field_last_value=None,  # every Phyllo endpoint is full refresh
        )