:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
from typing import Optional, cast
from products.warehouse_sources.sources.sdk import (
CanonicalDescriptions,
DataWarehouseSourceCategory,
ExternalDataSourceType,
FieldType,
ReleaseStatus,
ResumableSource,
ResumableSourceManager,
SourceConfig,
SourceFieldInputConfig,
SourceFieldInputConfigType,
SourceInputs,
SourceRegistry,
SourceResponse,
SourceSchema,
detect,
)
from products.warehouse_sources.sources.unstructured._config import UnstructuredSourceConfig
from products.warehouse_sources.sources.unstructured.settings import (
ENDPOINTS,
INCREMENTAL_FIELDS,
UNSTRUCTURED_ENDPOINTS,
)
from products.warehouse_sources.sources.unstructured.unstructured import (
UnstructuredResumeConfig,
unstructured_source,
validate_credentials as validate_unstructured_credentials,
)
@SourceRegistry.register
class UnstructuredSource(ResumableSource[UnstructuredSourceConfig, UnstructuredResumeConfig]):
codebase_detection = detect(
env=("^UNSTRUCTURED_",), npm=("unstructured-client",), pypi=("unstructured", "unstructured-client")
)
supported_versions = ("v1",)
default_version = "v1"
api_docs_url = "https://docs.unstructured.io/api-reference"
lists_tables_without_credentials = True # static endpoint catalog — safe for public docs
@property
def source_type(self) -> ExternalDataSourceType:
return ExternalDataSourceType.UNSTRUCTURED
@property
def connection_host_fields(self) -> list[str]:
# `base_url` is where the stored API key is sent; retargeting it must re-require the key so it
# can't be exfiltrated to another host.
return ["base_url"]
@property
def get_source_config(self) -> SourceConfig:
return SourceConfig(
name=ExternalDataSourceType.UNSTRUCTURED,
category=DataWarehouseSourceCategory.ENGINEERING___MONITORING,
label="Unstructured",
releaseStatus=ReleaseStatus.BETA,
caption="""Enter your Unstructured API key to pull your [Unstructured](https://unstructured.io) document-pipeline metadata into the PostHog Data warehouse.
Generate an API key in the [Unstructured platform dashboard](https://platform.unstructuredapp.io). The key has account-wide read access, so no extra scopes are needed.
Leave **API host** blank unless Unstructured provisioned your account with a custom API URL.""",
iconPath="/static/services/unstructured.svg",
docsUrl="https://posthog.com/docs/cdp/sources/unstructured",
keywords=["etl", "documents", "rag", "workflows"],
fields=cast(
list[FieldType],
[
SourceFieldInputConfig(
name="api_key",
label="API key",
type=SourceFieldInputConfigType.PASSWORD,
required=True,
placeholder="",
secret=True,
),
SourceFieldInputConfig(
name="base_url",
label="API host",
type=SourceFieldInputConfigType.TEXT,
required=False,
placeholder="https://platform.unstructuredapp.io",
secret=False,
),
],
),
)
def get_canonical_descriptions(self) -> CanonicalDescriptions:
from products.warehouse_sources.sources.unstructured.canonical_descriptions import CANONICAL_DESCRIPTIONS
return CANONICAL_DESCRIPTIONS
def get_non_retryable_errors(self) -> dict[str, str | None]:
# A bad or revoked key raises an HTTPError from `raise_for_status()`. The API host is
# user-configurable, so match on the stable status text rather than a fixed host.
return {
"401 Client Error: Unauthorized for url": "Your Unstructured API key is invalid or has been revoked. Generate a new key in the Unstructured platform dashboard, then reconnect.",
"403 Client Error: Forbidden for url": "Your Unstructured API key is not permitted to read this data. Check the key in the Unstructured platform dashboard, then reconnect.",
}
def get_schemas(
self,
config: UnstructuredSourceConfig,
team_id: int,
with_counts: bool = False,
names: list[str] | None = None,
force_refresh: bool = False,
api_version: str | None = None,
) -> list[SourceSchema]:
schemas = [
SourceSchema(
name=endpoint,
supports_incremental=INCREMENTAL_FIELDS.get(endpoint, None) is not None,
supports_append=INCREMENTAL_FIELDS.get(endpoint, None) is not None,
incremental_fields=INCREMENTAL_FIELDS.get(endpoint, []),
should_sync_default=UNSTRUCTURED_ENDPOINTS[endpoint].should_sync_default,
)
for endpoint in ENDPOINTS
]
if names is not None:
names_set = set(names)
schemas = [s for s in schemas if s.name in names_set]
return schemas
def validate_credentials(
self,
config: UnstructuredSourceConfig,
team_id: int,
schema_name: Optional[str] = None,
api_version: str | None = None,
) -> tuple[bool, str | None]:
return validate_unstructured_credentials(config.base_url, config.api_key, team_id)
def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[UnstructuredResumeConfig]:
return ResumableSourceManager[UnstructuredResumeConfig](inputs, UnstructuredResumeConfig)
def source_for_pipeline(
self,
config: UnstructuredSourceConfig,
resumable_source_manager: ResumableSourceManager[UnstructuredResumeConfig],
inputs: SourceInputs,
) -> SourceResponse:
return unstructured_source(
base_url=config.base_url,
api_key=config.api_key,
endpoint=inputs.schema_name,
logger=inputs.logger,
resumable_source_manager=resumable_source_manager,
team_id=inputs.team_id,
should_use_incremental_field=inputs.should_use_incremental_field,
db_incremental_field_last_value=inputs.db_incremental_field_last_value,
)