Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

5.2 KB
Raw Blame
from typing import cast

from products.warehouse_sources.sources.qdrant._config import QdrantSourceConfig
from products.warehouse_sources.sources.qdrant.canonical_descriptions import CANONICAL_DESCRIPTIONS
from products.warehouse_sources.sources.qdrant.qdrant import QdrantResumeConfig, qdrant_source, validate_credentials
from products.warehouse_sources.sources.qdrant.settings import (
    API_VERSION,
    AUTH_ERROR,
    ENDPOINTS,
    INCREMENTAL_FIELDS,
    PERMISSION_ERROR,
)
from products.warehouse_sources.sources.sdk import (
    CanonicalDescriptions,
    DataWarehouseSourceCategory,
    ExternalDataSourceType,
    FieldType,
    ReleaseStatus,
    ResumableSource,
    ResumableSourceManager,
    SourceConfig,
    SourceFieldInputConfig,
    SourceFieldInputConfigType,
    SourceInputs,
    SourceRegistry,
    SourceResponse,
    SourceSchema,
    build_endpoint_schemas,
    detect,
)


@SourceRegistry.register
class QdrantSource(ResumableSource[QdrantSourceConfig, QdrantResumeConfig]):
    codebase_detection = detect(
        env=("^QDRANT_",),
        npm=("@qdrant/js-client-rest", "@qdrant/qdrant-js"),
        pypi=("qdrant-client",),
        cargo=("qdrant-client", "qdrant", "qdrant-edge", "qdrant-rust-stemmers"),
        go=(
            "github.com/qdrant/go-client",
            "github.com/qdrant/qdrant-cloud-buf-plugins",
            "github.com/qdrant/qdrant-cloud-public-api",
            "github.com/qdrant/qdrant-helm",
        ),
        maven=("io.qdrant:client",),
    )

    lists_tables_without_credentials = True
    supported_versions = (API_VERSION,)
    default_version = API_VERSION
    api_docs_url = "https://qdrant.tech/documentation/cloud-api/"

    @property
    def source_type(self) -> ExternalDataSourceType:
        return ExternalDataSourceType.QDRANT

    def get_non_retryable_errors(self) -> dict[str, str | None]:
        return {"401 Client Error": AUTH_ERROR, "403 Client Error": PERMISSION_ERROR}

    def get_canonical_descriptions(self) -> CanonicalDescriptions:
        return CANONICAL_DESCRIPTIONS

    def get_schemas(
        self,
        config: QdrantSourceConfig,
        team_id: int,
        with_counts: bool = False,
        names: list[str] | None = None,
        force_refresh: bool = False,
        api_version: str | None = None,
    ) -> list[SourceSchema]:
        # The metrics window re-reads points before the watermark, so append would duplicate them.
        return build_endpoint_schemas(ENDPOINTS, INCREMENTAL_FIELDS, names, merge_only=INCREMENTAL_FIELDS.keys())

    def validate_credentials(
        self,
        config: QdrantSourceConfig,
        team_id: int,
        schema_name: str | None = None,
        api_version: str | None = None,
    ) -> tuple[bool, str | None]:
        return validate_credentials(config, schema_name)

    def get_resumable_source_manager(self, inputs: SourceInputs) -> ResumableSourceManager[QdrantResumeConfig]:
        return ResumableSourceManager(inputs, QdrantResumeConfig)

    def source_for_pipeline(
        self,
        config: QdrantSourceConfig,
        resumable_source_manager: ResumableSourceManager[QdrantResumeConfig],
        inputs: SourceInputs,
    ) -> SourceResponse:
        return qdrant_source(
            config,
            inputs.schema_name,
            inputs.team_id,
            inputs.job_id,
            resumable_source_manager,
            should_use_incremental_field=inputs.should_use_incremental_field,
            db_incremental_field_last_value=inputs.db_incremental_field_last_value,
        )

    @property
    def get_source_config(self) -> SourceConfig:
        return SourceConfig(
            name=ExternalDataSourceType.QDRANT,
            category=DataWarehouseSourceCategory.DATABASES,
            label="Qdrant",
            iconPath="/static/services/qdrant.png",
            keywords=["vector", "database", "qdrant", "cloud"],
            caption=(
                "Import Qdrant Cloud infrastructure metadata. "
                "Create a Cloud Management Key in the Cloud Console under Access Management > Cloud Management Keys. "
                "Grant `read:clusters`, `read:backups`, `read:backup_schedules`, `read:payment_information`, and `read:users` for the tables you select. "
                "Copy your account ID from the Cloud Console. Vector data and collections are not included."
            ),
            fields=cast(
                list[FieldType],
                [
                    SourceFieldInputConfig(
                        name="api_key",
                        label="Cloud Management Key",
                        type=SourceFieldInputConfigType.PASSWORD,
                        placeholder="",
                        required=True,
                        secret=True,
                    ),
                    SourceFieldInputConfig(
                        name="account_id",
                        label="Account ID",
                        type=SourceFieldInputConfigType.TEXT,
                        placeholder="00000000-0000-0000-0000-000000000000",
                        required=True,
                        secret=False,
                    ),
                ],
            ),
            releaseStatus=ReleaseStatus.BETA,
        )