Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.4 KB
Raw Blame
from typing import Optional, cast

from products.warehouse_sources.sources.firecrawl._config import FirecrawlSourceConfig
from products.warehouse_sources.sources.firecrawl.firecrawl import (
    FirecrawlResumeConfig,
    firecrawl_source,
    validate_credentials as validate_firecrawl_credentials,
)
from products.warehouse_sources.sources.firecrawl.settings import ENDPOINTS, FIRECRAWL_ENDPOINTS
from products.warehouse_sources.sources.sdk import (
    CanonicalDescriptions,
    DataWarehouseSourceCategory,
    ExternalDataSourceType,
    FieldType,
    ReleaseStatus,
    ResumableSource,
    ResumableSourceManager,
    SourceConfig,
    SourceFieldInputConfig,
    SourceFieldInputConfigType,
    SourceInputs,
    SourceRegistry,
    SourceResponse,
    SourceSchema,
    detect,
)

_ENDPOINT_DESCRIPTIONS: dict[str, str] = {
    "team_activity": "Job activity log. Firecrawl only retains the last 24 hours, so sync frequently to accumulate history - older activity cannot be backfilled.",
    "monitor_checks": "Change-detection runs, one row per check across every monitor. Off by default: it fans out one request per monitor.",
}


@SourceRegistry.register
class FirecrawlSource(ResumableSource[FirecrawlSourceConfig, FirecrawlResumeConfig]):
    codebase_detection = detect(
        env=("^FIRECRAWL_",),
        npm=("@mendable/firecrawl-js", "firecrawl"),
        pypi=("firecrawl-py", "firecrawl"),
        cargo=("firecrawl", "firecrawl_rs"),
        go=("github.com/firecrawl/firecrawl-go", "github.com/mendableai/firecrawl-go"),
        maven=("com.firecrawl:firecrawl-java",),
        rubygems=("firecrawl-sdk",),
    )

    lists_tables_without_credentials = True  # static endpoint catalog - safe for public docs
    supported_versions = ("v2",)
    default_version = "v2"
    api_docs_url = "https://docs.firecrawl.dev/api-reference"

    @property
    def source_type(self) -> ExternalDataSourceType:
        return ExternalDataSourceType.FIRECRAWL

    @property
    def get_source_config(self) -> SourceConfig:
        return SourceConfig(
            name=ExternalDataSourceType.FIRECRAWL,
            category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
            label="Firecrawl",
            releaseStatus=ReleaseStatus.GA,
            caption="""Enter your Firecrawl API key to pull your Firecrawl account activity and usage into the PostHog Data warehouse.

You can create an API key in your [Firecrawl dashboard](https://www.firecrawl.dev/app/api-keys). A single key grants access to all of the tables below.""",
            iconPath="/static/services/firecrawl.svg",
            docsUrl="https://posthog.com/docs/cdp/sources/firecrawl",
            keywords=["scraping", "crawling", "web data", "firecrawl"],
            fields=cast(
                list[FieldType],
                [
                    SourceFieldInputConfig(
                        name="api_key",
                        label="API key",
                        type=SourceFieldInputConfigType.PASSWORD,
                        required=True,
                        placeholder="fc-...",
                        secret=True,
                    ),
                ],
            ),
        )

    def get_canonical_descriptions(self) -> CanonicalDescriptions:
        from products.warehouse_sources.sources.firecrawl.canonical_descriptions import CANONICAL_DESCRIPTIONS

        return CANONICAL_DESCRIPTIONS

    def get_non_retryable_errors(self) -> dict[str, str | None]:
        return {
            # An invalid or revoked key surfaces as an HTTPError from raise_for_status(). Retrying can
            # never satisfy a credential problem, so stop the sync. Match the stable status text plus
            # base host, not the per-request path.
            "401 Client Error: Unauthorized for url: https://api.firecrawl.dev": "Your Firecrawl API key is invalid or has been revoked. Create a new key in your Firecrawl dashboard, then reconnect.",
            "403 Client Error: Forbidden for url: https://api.firecrawl.dev": "Your Firecrawl API key does not have access to this data. Check the key in your Firecrawl dashboard, then reconnect.",
        }

    def get_schemas(
        self,
        config: FirecrawlSourceConfig,
        team_id: int,
        with_counts: bool = False,
        names: list[str] | None = None,
        force_refresh: bool = False,
        api_version: str | None = None,
    ) -> list[SourceSchema]:
        def _build_schema(endpoint: str) -> SourceSchema:
            endpoint_config = FIRECRAWL_ENDPOINTS[endpoint]
            # Every endpoint is full-refresh: Firecrawl has no server-side timestamp filter to drive a
            # genuine incremental sync (see settings.py).
            return SourceSchema(
                name=endpoint,
                supports_incremental=False,
                supports_append=False,
                incremental_fields=[],
                should_sync_default=endpoint_config.should_sync_default,
                description=_ENDPOINT_DESCRIPTIONS.get(endpoint),
                detected_primary_keys=endpoint_config.primary_keys,
            )

        schemas = [_build_schema(endpoint) for endpoint in ENDPOINTS]
        if names is not None:
            names_set = set(names)
            schemas = [s for s in schemas if s.name in names_set]
        return schemas

    def validate_credentials(
        self,
        config: FirecrawlSourceConfig,
        team_id: int,
        schema_name: Optional[str] = None,
        api_version: str | None = None,
    ) -> tuple[bool, str | None]:
        # A Firecrawl key is all-or-nothing (no per-endpoint scopes), so the same token probe covers
        # both source-create (schema_name=None) and the per-schema check.
        if validate_firecrawl_credentials(config.api_key):
            return True, None
        return False, "Invalid Firecrawl API key"

    def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[FirecrawlResumeConfig]:
        return ResumableSourceManager[FirecrawlResumeConfig](inputs, FirecrawlResumeConfig)

    def source_for_pipeline(
        self,
        config: FirecrawlSourceConfig,
        resumable_source_manager: ResumableSourceManager[FirecrawlResumeConfig],
        inputs: SourceInputs,
    ) -> SourceResponse:
        return firecrawl_source(
            api_key=config.api_key,
            endpoint=inputs.schema_name,
            team_id=inputs.team_id,
            job_id=inputs.job_id,
            resumable_source_manager=resumable_source_manager,
        )