Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

6.9 KB
Raw Blame
from typing import Optional, cast

from products.warehouse_sources.sources.sdk import (
    CanonicalDescriptions,
    DataWarehouseSourceCategory,
    ExternalDataSourceType,
    FieldType,
    ReleaseStatus,
    ResumableSource,
    ResumableSourceManager,
    SourceConfig,
    SourceFieldInputConfig,
    SourceFieldInputConfigType,
    SourceInputs,
    SourceRegistry,
    SourceResponse,
    SourceSchema,
    detect,
)
from products.warehouse_sources.sources.stockdata._config import StockDataSourceConfig
from products.warehouse_sources.sources.stockdata.settings import STOCKDATA_ENDPOINTS
from products.warehouse_sources.sources.stockdata.stockdata import (
    StockDataResumeConfig,
    stockdata_source,
    validate_credentials as validate_stockdata_credentials,
)


@SourceRegistry.register
class StockDataSource(ResumableSource[StockDataSourceConfig, StockDataResumeConfig]):
    codebase_detection = detect(env=("^(STOCK_DATA|STOCKDATA)_",))

    lists_tables_without_credentials = True  # static endpoint catalog — safe for public docs
    supported_versions = ("v1",)
    default_version = "v1"
    api_docs_url = "https://www.stockdata.org/documentation"

    @property
    def source_type(self) -> ExternalDataSourceType:
        return ExternalDataSourceType.STOCKDATA

    def get_non_retryable_errors(self) -> dict[str, str | None]:
        return {
            "401 Client Error: Unauthorized for url: https://api.stockdata.org": "Your StockData.org API token is invalid. Generate a new token in your StockData.org dashboard, then reconnect.",
            "402 Client Error: Payment Required for url: https://api.stockdata.org": "Your StockData.org plan's usage limit has been reached. Upgrade your plan or wait for the quota to reset, then resync.",
            "403 Client Error: Forbidden for url: https://api.stockdata.org": "Your StockData.org plan does not grant access to this data (e.g. dividends, splits, and the news stats tables require a Standard or higher plan). Upgrade your plan or deselect the restricted tables, then resync.",
            # The price and entity search tables need at least one symbol; without one the sync can never make
            # progress, so fail permanently with a fix-it message rather than retrying.
            "StockData.org API error [missing_symbols]": "One or more of the selected tables (quote, EOD, intraday, dividends, splits, entity search) requires symbols. Add symbols to the source configuration, then resync.",
        }

    def get_canonical_descriptions(self) -> CanonicalDescriptions:
        from products.warehouse_sources.sources.stockdata.canonical_descriptions import CANONICAL_DESCRIPTIONS

        return CANONICAL_DESCRIPTIONS

    def get_schemas(
        self,
        config: StockDataSourceConfig,
        team_id: int,
        with_counts: bool = False,
        names: list[str] | None = None,
        force_refresh: bool = False,
        api_version: str | None = None,
    ) -> list[SourceSchema]:
        schemas = [
            SourceSchema(
                name=endpoint.name,
                # Only news and the news stats time series (published_after) and the EOD/intraday
                # price feeds (date_from) expose a server-side date filter; the rest are full refresh only.
                supports_incremental=bool(endpoint.incremental_fields),
                supports_append=bool(endpoint.incremental_fields),
                incremental_fields=list(endpoint.incremental_fields),
                description=endpoint.description,
            )
            for endpoint in STOCKDATA_ENDPOINTS.values()
        ]

        if names is not None:
            names_set = set(names)
            schemas = [s for s in schemas if s.name in names_set]

        return schemas

    def validate_credentials(
        self,
        config: StockDataSourceConfig,
        team_id: int,
        schema_name: Optional[str] = None,
        api_version: str | None = None,
    ) -> tuple[bool, str | None]:
        return validate_stockdata_credentials(config.api_token)

    def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[StockDataResumeConfig]:
        return ResumableSourceManager[StockDataResumeConfig](inputs, StockDataResumeConfig)

    def source_for_pipeline(
        self,
        config: StockDataSourceConfig,
        resumable_source_manager: ResumableSourceManager[StockDataResumeConfig],
        inputs: SourceInputs,
    ) -> SourceResponse:
        return stockdata_source(
            api_token=config.api_token,
            endpoint=inputs.schema_name,
            team_id=inputs.team_id,
            job_id=inputs.job_id,
            resumable_source_manager=resumable_source_manager,
            symbols=config.symbols,
            db_incremental_field_last_value=inputs.db_incremental_field_last_value
            if inputs.should_use_incremental_field
            else None,
        )

    @property
    def get_source_config(self) -> SourceConfig:
        return SourceConfig(
            name=ExternalDataSourceType.STOCKDATA,
            category=DataWarehouseSourceCategory.FINANCE___ACCOUNTING,
            label="StockData.org",
            releaseStatus=ReleaseStatus.BETA,
            caption="""Enter your StockData.org API token to pull market news with sentiment, news stats, entity and news source metadata, plus quote, end-of-day, intraday, dividend, and split price data into the PostHog Data warehouse.

You can find your API token in your [StockData.org dashboard](https://www.stockdata.org/dashboard).

Add one or more comma-separated **symbols** (e.g. `AAPL,MSFT,TSLA`) to sync the price tables (quote, EOD, intraday, dividends, splits) and the entity search table — those endpoints require at least one symbol. The news and news stats tables work without symbols (all market news) and are filtered to your symbols when they are set.

Note: StockData.org plans are limited by daily request quotas (100 requests/day on the free plan), and some tables (dividends, splits, news stats) require a Standard or higher plan.""",
            iconPath="/static/services/stockdata.png",
            docsUrl="https://posthog.com/docs/cdp/sources/stockdata",
            fields=cast(
                list[FieldType],
                [
                    SourceFieldInputConfig(
                        name="api_token",
                        label="API token",
                        type=SourceFieldInputConfigType.PASSWORD,
                        required=True,
                        placeholder="",
                        secret=True,
                    ),
                    SourceFieldInputConfig(
                        name="symbols",
                        sync_setting=True,
                        label="Symbols",
                        type=SourceFieldInputConfigType.TEXT,
                        required=False,
                        placeholder="AAPL,MSFT,TSLA",
                        secret=False,
                    ),
                ],
            ),
        )