:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from typing import Optional, cast
from products.warehouse_sources.sources.sdk import (
CanonicalDescriptions,
DataWarehouseSourceCategory,
ExternalDataSourceType,
FieldType,
ReleaseStatus,
ResumableSource,
ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
SourceInputs,
SourceRegistry,
SourceResponse,
SourceSchema,
build_endpoint_schemas,
detect,
)
from products.warehouse_sources.sources.twitter._config import TwitterSourceConfig
from products.warehouse_sources.sources.twitter.settings import API_VERSION_2, ENDPOINTS, INCREMENTAL_FIELDS
from products.warehouse_sources.sources.twitter.twitter import (
TwitterResumeConfig,
endpoint_permissions,
twitter_source,
validate_credentials as validate_twitter_credentials,
)
@SourceRegistry.register
class TwitterSource(ResumableSource[TwitterSourceConfig, TwitterResumeConfig]):
codebase_detection = detect(
env=("^TWITTER_",),
script_hosts=("platform.twitter.com",),
npm=("@xdevplatform/xdk", "twitter-api-sdk", "twitter-api-v2"),
pypi=("xdk", "tweepy"),
rubygems=("twitter",),
)
lists_tables_without_credentials = True # static endpoint catalog — safe for public docs
supported_versions = (API_VERSION_2,)
default_version = API_VERSION_2
api_docs_url = "https://docs.x.com/x-api/introduction"
@property
def source_type(self) -> ExternalDataSourceType:
return ExternalDataSourceType.TWITTER
def get_non_retryable_errors(self) -> dict[str, str | None]:
return {
"401 Client Error: Unauthorized for url: https://api.x.com": "X rejected the bearer token. Generate a new one in the X developer portal and reconnect the source.",
"403 Client Error: Forbidden for url: https://api.x.com": "Your X app cannot read this endpoint. Check the project's API access and credit balance in the X developer portal, or stop syncing this table.",
# X's pay-per-use plan answers 402 once the project's prepaid credits run out, mirroring
# the check in `validate_credentials` — retrying a sync won't refill the balance.
"402 Client Error: Payment Required for url: https://api.x.com": "Your X developer project has run out of API credits. Add credits in the X developer portal, then try again.",
}
def get_canonical_descriptions(self) -> CanonicalDescriptions:
from products.warehouse_sources.sources.twitter.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
def get_schemas(
self,
config: TwitterSourceConfig,
team_id: int,
with_counts: bool = False,
names: list[str] | None = None,
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names)
def validate_credentials(
self,
config: TwitterSourceConfig,
team_id: int,
schema_name: Optional[str] = None,
api_version: str | None = None,
) -> tuple[bool, str | None]:
return validate_twitter_credentials(config.bearer_token, config.username)
def get_endpoint_permissions(
self, config: TwitterSourceConfig, team_id: int, endpoints: list[str], api_version: str | None = None
) -> dict[str, str | None]:
return endpoint_permissions(config.bearer_token, config.username, endpoints)
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[TwitterResumeConfig]:
return ResumableSourceManager[TwitterResumeConfig](inputs, TwitterResumeConfig)
def source_for_pipeline(
self,
config: TwitterSourceConfig,
resumable_source_manager: ResumableSourceManager[TwitterResumeConfig],
inputs: SourceInputs,
) -> SourceResponse:
return twitter_source(
bearer_token=config.bearer_token,
username=config.username,
endpoint=inputs.schema_name,
team_id=inputs.team_id,
job_id=inputs.job_id,
resumable_source_manager=resumable_source_manager,
should_use_incremental_field=inputs.should_use_incremental_field,
db_incremental_field_last_value=inputs.db_incremental_field_last_value
if inputs.should_use_incremental_field
else None,
)
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
name=ExternalDataSourceType.TWITTER,
category=DataWarehouseSourceCategory.COMMUNICATION,
label="Twitter",
keywords=["x", "x.com"],
caption=(
"Sync one X account's profile, posts, mentions and audience. "
"**X charges your own developer project for every post this source reads.** "
"Create an app in the [X developer portal](https://developer.x.com/en/portal/dashboard), "
"attach it to a project that has API credits, then copy the app's **Bearer Token**."
),
docsUrl="https://posthog.com/docs/cdp/sources/twitter",
iconPath="/static/services/twitter.png",
releaseStatus=ReleaseStatus.GA,
fields=cast(
list[FieldType],
[
SourceFieldInputConfig(
name="bearer_token",
label="Bearer token",
type=SourceFieldInputConfigType.PASSWORD,
required=True,
placeholder="",
secret=True,
),
SourceFieldInputConfig(
name="username",
label="Account handle",
type=SourceFieldInputConfigType.TEXT,
required=True,
placeholder="posthog",
secret=False,
),
],
),
)