Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

13.3 KB
Raw Blame
from collections.abc import Sequence
from dataclasses import dataclass, field
from unittest.mock import Mock

import pytest
from sqlalchemy.orm import Session, sessionmaker

from core.rag.extractor.entity.datasource_type import DatasourceType
from core.rag.extractor.entity.extract_setting import UploadFileExtractionInput
from core.rag.extractor.notion_extractor import NotionExtractor
from extensions.application_services.data_sources import build_data_source_credentials
from services.credentials.refresh import CredentialRefreshUnavailableError
from services.data_source.credential_gateway import (
    DatasourceCredentialNotFoundError,
    DatasourceCredentialRefreshError,
)
from services.data_source.entities.notion_import import NotionPageType
from services.knowledge.indexing.adapters.sources import (
    CompositeStoredSourceResolver,
    FileSourceAdapter,
    NotionSourceResolver,
    WebsiteSourceAdapter,
)
from services.knowledge.indexing.errors import (
    IndexingInputSourceError,
    SourceCredentialUnavailableError,
    UnsupportedStoredSourceError,
)
from services.knowledge.indexing.estimate import StoredSource
from services.knowledge.resource_scope import DatasetRef
from tests.helpers.unit.config_override import apply_config_overrides


def _source(
    source_type: str,
    source_info: dict[str, object] | None,
    *,
    document_id: str = "document-1",
) -> StoredSource:
    return StoredSource(
        document_ref=DatasetRef("workspace-1", "dataset-1").document(document_id),
        source_type=source_type,
        source_info=source_info,
        document_model="text_model",
    )


@dataclass
class RecordingUploads:
    uploads: dict[str, UploadFileExtractionInput]
    calls: list[tuple[str, tuple[str, ...]]] = field(default_factory=list)

    def get_by_ids(self, *, workspace_id: str, file_ids: Sequence[str]) -> dict[str, UploadFileExtractionInput]:
        self.calls.append((workspace_id, tuple(file_ids)))
        return {file_id: self.uploads[file_id] for file_id in file_ids if file_id in self.uploads}


@dataclass
class RecordingActorCredentials:
    credentials: dict[str, object] | None = None
    error: Exception | None = None
    calls: list[dict[str, object]] = field(default_factory=list)

    def resolve(self, **kwargs: object) -> dict[str, object]:
        self.calls.append(kwargs)
        if self.error is not None:
            raise self.error
        return self.credentials or {}


@dataclass
class RecordingStoredCredentials:
    credentials: dict[str, object] | None = None
    error: Exception | None = None
    calls: list[dict[str, object]] = field(default_factory=list)

    def resolve_for_document(self, **kwargs: object) -> dict[str, object]:
        self.calls.append(kwargs)
        if self.error is not None:
            raise self.error
        return self.credentials or {}


def _notion_resolver(
    *,
    actor: RecordingActorCredentials | None = None,
    stored: RecordingStoredCredentials | None = None,
    integration_token: str | None = None,
) -> NotionSourceResolver:
    return NotionSourceResolver(
        actor_credentials=actor or RecordingActorCredentials({"integration_secret": "actor-secret"}),
        stored_credentials=stored or RecordingStoredCredentials({"integration_secret": "stored-secret"}),
        integration_token=integration_token,
    )


def test_datasource_adapters_resolve_file_website_and_notion_inputs() -> None:
    upload = UploadFileExtractionInput(
        id="file-1",
        tenant_id="workspace-1",
        key="file.txt",
        created_by="account-1",
    )
    uploads = RecordingUploads({upload.id: upload})
    notion = _notion_resolver()
    resolver = CompositeStoredSourceResolver(
        adapters={
            "upload_file": FileSourceAdapter(uploads=uploads),
            "website_crawl": WebsiteSourceAdapter(),
            "notion_import": notion,
        }
    )

    file_extraction = resolver.resolve(
        _source("upload_file", {"upload_file_id": upload.id}),
    )
    website_extraction = resolver.resolve(
        _source(
            "website_crawl",
            {
                "provider": "firecrawl",
                "job_id": "job-1",
                "url": "https://example.com",
                "mode": "crawl",
                "only_main_content": True,
            },
        ),
    )
    notion_extraction = resolver.resolve(
        _source(
            "notion_import",
            {
                "credential_id": "credential-1",
                "notion_workspace_id": "notion-workspace-1",
                "notion_page_id": "notion-page-1",
                "type": "page",
            },
        ),
    )

    assert file_extraction.datasource_type == DatasourceType.FILE
    assert file_extraction.upload_file == upload
    assert uploads.calls == [("workspace-1", ("file-1",))]
    assert website_extraction.website_info is not None
    assert website_extraction.website_info.tenant_id == "workspace-1"
    assert website_extraction.website_info.only_main_content is True
    assert notion_extraction.notion_info is not None
    assert notion_extraction.notion_info.notion_access_token == "stored-secret"


def test_stored_notion_resolution_uses_trusted_document_owner_chain() -> None:
    actor = RecordingActorCredentials({"integration_secret": "actor-secret"})
    stored = RecordingStoredCredentials({"integration_secret": "stored-secret"})
    resolver = _notion_resolver(actor=actor, stored=stored)

    extraction = resolver.resolve(
        _source(
            "notion_import",
            {
                "credential_id": "private-credential",
                "notion_workspace_id": "notion-workspace-1",
                "notion_page_id": "notion-page-1",
                "type": "database",
            },
        ),
    )

    assert actor.calls == []
    assert stored.calls == [
        {
            "workspace_id": "workspace-1",
            "dataset_id": "dataset-1",
            "document_id": "document-1",
            "credential_id": "private-credential",
            "provider": "notion_datasource",
            "plugin_id": "langgenius/notion_datasource",
        }
    ]
    assert extraction.notion_info is not None
    assert extraction.notion_info.notion_page_type == NotionPageType.DATABASE.value


@pytest.mark.parametrize(
    ("credential_id", "credentials", "error", "expected_token"),
    [
        (None, {"integration_secret": "default-secret"}, None, "environment-secret"),
        ("credential-1", {"integration_secret": "saved-secret"}, None, "saved-secret"),
        ("deleted-credential", {}, DatasourceCredentialNotFoundError(), "environment-secret"),
        ("credential-1", {}, DatasourceCredentialRefreshError("credential-1"), "environment-secret"),
        ("credential-1", {}, CredentialRefreshUnavailableError(), "environment-secret"),
        ("credential-1", {}, None, "environment-secret"),
        ("credential-1", {"integration_secret": ""}, None, "environment-secret"),
        ("credential-1", {"integration_secret": 42}, None, "environment-secret"),
    ],
    ids=[
        "legacy",
        "saved",
        "deleted",
        "refresh-failed",
        "refresh-unavailable",
        "missing-token",
        "empty-token",
        "invalid-token",
    ],
)
@pytest.mark.parametrize("integration_token", ["environment-secret", None, ""])
def test_stored_notion_preview_and_indexing_use_same_integration(
    sqlite_session_factory: sessionmaker[Session],
    monkeypatch: pytest.MonkeyPatch,
    credential_id: str | None,
    credentials: dict[str, object],
    error: Exception | None,
    expected_token: str,
    integration_token: str | None,
) -> None:
    apply_config_overrides(monkeypatch, NOTION_INTEGRATION_TOKEN=integration_token)
    stored = RecordingStoredCredentials(credentials, error=error)
    resolver = _notion_resolver(stored=stored)
    indexing_credentials = Mock(return_value=credentials, side_effect=error)
    providers = build_data_source_credentials(database_client=sqlite_session_factory).providers
    monkeypatch.setattr(
        providers,
        "get_datasource_credentials",
        indexing_credentials,
    )
    source = _source(
        "notion_import",
        {
            "credential_id": credential_id,
            "notion_workspace_id": "notion-workspace-1",
            "notion_page_id": "notion-page-1",
            "type": "page",
        },
    )

    def indexing_extractor() -> NotionExtractor:
        return NotionExtractor(
            notion_workspace_id="notion-workspace-1",
            notion_obj_id="notion-page-1",
            notion_page_type="page",
            tenant_id="workspace-1",
            credential_id=credential_id,
            notion_token_loader=lambda: providers.get_stored_notion_access_token(
                tenant_id="workspace-1", credential_id=credential_id
            ),
        )

    if isinstance(error, CredentialRefreshUnavailableError) and not integration_token:
        with pytest.raises(CredentialRefreshUnavailableError):
            resolver.resolve(source)
        with pytest.raises(CredentialRefreshUnavailableError):
            indexing_extractor()
    elif expected_token == "environment-secret" and not integration_token:
        with pytest.raises(SourceCredentialUnavailableError):
            resolver.resolve(source)
        with pytest.raises(ValueError, match="Must specify `integration_token`"):
            indexing_extractor()
    else:
        preview = resolver.resolve(source)
        extractor = indexing_extractor()
        assert preview.notion_info is not None
        assert preview.notion_info.notion_access_token == extractor._notion_access_token == expected_token

    expected_ids: list[str] = [credential_id] if credential_id else []
    assert [call["credential_id"] for call in stored.calls] == expected_ids
    assert [call.kwargs["credential_id"] for call in indexing_credentials.call_args_list] == expected_ids


def test_notion_resolution_falls_back_to_configured_integration_token() -> None:
    stored = RecordingStoredCredentials(error=DatasourceCredentialNotFoundError())
    resolver = _notion_resolver(stored=stored, integration_token="environment-secret")

    extraction = resolver.resolve(
        _source(
            "notion_import",
            {
                "notion_workspace_id": "notion-workspace-1",
                "notion_page_id": "notion-page-1",
                "type": "page",
            },
        ),
    )

    assert extraction.notion_info is not None
    assert extraction.notion_info.notion_access_token == "environment-secret"


def test_new_notion_selection_uses_actor_scoped_credential() -> None:
    actor = RecordingActorCredentials({"integration_secret": "actor-secret"})
    stored = RecordingStoredCredentials({"integration_secret": "stored-secret"})
    resolver = _notion_resolver(actor=actor, stored=stored)

    extraction = resolver.resolve_selection(
        workspace_id="workspace-1",
        actor_id="account-1",
        credential_id="credential-1",
        notion_workspace_id="notion-workspace-1",
        page_id="notion-page-1",
        page_type=NotionPageType.PAGE,
        document_model="text_model",
    )

    assert stored.calls == []
    assert actor.calls[0]["actor_id"] == "account-1"
    assert extraction.notion_info is not None
    assert extraction.notion_info.notion_access_token == "actor-secret"


def test_new_notion_selection_does_not_fall_back_when_actor_cannot_access_credential() -> None:
    actor = RecordingActorCredentials(error=DatasourceCredentialNotFoundError())
    resolver = _notion_resolver(actor=actor, integration_token="environment-secret")

    with pytest.raises(DatasourceCredentialNotFoundError):
        resolver.resolve_selection(
            workspace_id="workspace-1",
            actor_id="account-1",
            credential_id="private-credential",
            notion_workspace_id="notion-workspace-1",
            page_id="notion-page-1",
            page_type=NotionPageType.PAGE,
            document_model="text_model",
        )


def test_invalid_or_unknown_sources_are_rejected_at_the_adapter_boundary() -> None:
    resolver = CompositeStoredSourceResolver(adapters={"website_crawl": WebsiteSourceAdapter()})

    with pytest.raises(IndexingInputSourceError, match="provider"):
        resolver.resolve(
            _source("website_crawl", {"job_id": "job", "url": "url", "mode": "crawl"}),
        )
    with pytest.raises(UnsupportedStoredSourceError, match="mystery"):
        resolver.resolve(_source("mystery", {}))


def test_formal_notion_extraction_carries_source_update_scope_but_preview_does_not() -> None:
    source = _source(
        "notion_import",
        {
            "credential_id": "credential-1",
            "notion_workspace_id": "notion-workspace-1",
            "notion_page_id": "notion-page-1",
            "type": "page",
        },
    )
    resolver = CompositeStoredSourceResolver(adapters={"notion_import": _notion_resolver()})
    preview = resolver.resolve(source)
    formal = resolver.resolve_for_indexing(source)
    assert preview.notion_info is not None
    assert formal.notion_info is not None
    assert preview.notion_info.document is None
    document = formal.notion_info.document
    assert document is not None
    assert (document.tenant_id, document.dataset_id, document.id) == (
        source.document_ref.dataset.tenant_id,
        source.document_ref.dataset.dataset_id,
        source.document_ref.document_id,
    )
    assert document.data_source_info_dict == source.source_info