Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(rag-pipeline): mark credential-free datasources as authorized (#39772)

ccl125d5a8ace1c4c1
diff --git a/api/controllers/console/app/workflow_run.py b/api/controllers/console/app/workflow_run.py
index bc842dbf6b4..fb1a0361ce8 100644
--- a/api/controllers/console/app/workflow_run.py
+++ b/api/controllers/console/app/workflow_run.py
@@ -37,7 +37,7 @@ from fields.workflow_run_fields import (
from graphon.enums import WorkflowExecutionStatus
from libs.archive_storage import ArchiveStorageNotConfiguredError, get_archive_storage
from libs.custom_inputs import time_duration
-from libs.helper import uuid_value
+from libs.helper import dump_response, uuid_value
from libs.login import login_required
from models import Account, App, AppMode, WorkflowArchiveLog, WorkflowRunTriggeredFrom
from models.workflow import WorkflowRun
@@ -232,14 +232,17 @@ class WorkflowRunExportApi(Resource):
expires_in=EXPORT_SIGNED_URL_EXPIRE_SECONDS,
)
expires_at = datetime.now(UTC) + timedelta(seconds=EXPORT_SIGNED_URL_EXPIRE_SECONDS)
- response = WorkflowRunExportResponse.model_validate(
- {
- "status": "success",
- "presigned_url": presigned_url,
- "presigned_url_expires_at": expires_at.isoformat(),
- }
+ return (
+ dump_response(
+ WorkflowRunExportResponse,
+ {
+ "status": "success",
+ "presigned_url": presigned_url,
+ "presigned_url_expires_at": expires_at.isoformat(),
+ },
+ ),
+ 200,
)
- return response.model_dump(mode="json"), 200
@console_ns.route("/apps/<uuid:app_id>/advanced-chat/workflow-runs/count")
diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.py
index edb74bf50cc..44af59123d1 100644
--- a/api/controllers/console/datasets/datasets_document.py
+++ b/api/controllers/console/datasets/datasets_document.py
@@ -1073,56 +1073,59 @@ class DocumentApi(DocumentResource):
metadata_fields = {"doc_type", "doc_metadata"}
if metadata == "only":
- response = DocumentDetailResponse.model_validate(
- {
- "id": document.id,
- "doc_type": document.doc_type,
- "doc_metadata": document.get_doc_metadata_details(session=session),
- }
+ return (
+ dump_response(
+ DocumentDetailResponse,
+ {
+ "id": document.id,
+ "doc_type": document.doc_type,
+ "doc_metadata": document.get_doc_metadata_details(session=session),
+ },
+ include={"id", *metadata_fields},
+ exclude_unset=True,
+ ),
+ 200,
)
- return response.model_dump(mode="json", include={"id", *metadata_fields}, exclude_unset=True), 200
dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, session)
document_process_rule = document.get_dataset_process_rule(session=session)
document_process_rules: Mapping[str, Any] = document_process_rule.to_dict() if document_process_rule else {}
segment_count = document.get_segment_count(session=session)
- response = DocumentDetailResponse.model_validate(
- {
- "id": document.id,
- "position": document.position,
- "data_source_type": document.data_source_type,
- "data_source_info": document.data_source_info_dict,
- "data_source_detail_dict": document.get_data_source_detail_dict(session=session),
- "dataset_process_rule_id": document.dataset_process_rule_id,
- "dataset_process_rule": dataset_process_rules,
- "document_process_rule": document_process_rules,
- "name": document.name,
- "created_from": document.created_from,
- "created_by": document.created_by,
- "created_at": int(document.created_at.timestamp()),
- "tokens": document.tokens,
- "indexing_status": document.indexing_status,
- "completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,
- "updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,
- "indexing_latency": document.indexing_latency,
- "error": document.error,
- "enabled": document.enabled,
- "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,
- "disabled_by": document.disabled_by,
- "archived": document.archived,
- "doc_type": document.doc_type,
- "doc_metadata": document.get_doc_metadata_details(session=session),
- "segment_count": segment_count,
- "average_segment_length": (document.word_count or 0) // segment_count if segment_count else 0,
- "hit_count": document.get_hit_count(session=session),
- "display_status": document.display_status,
- "doc_form": document.doc_form,
- "doc_language": document.doc_language,
- "need_summary": document.need_summary if document.need_summary is not None else False,
- }
- )
+ response_payload = {
+ "id": document.id,
+ "position": document.position,
+ "data_source_type": document.data_source_type,
+ "data_source_info": document.data_source_info_dict,
+ "data_source_detail_dict": document.get_data_source_detail_dict(session=session),
+ "dataset_process_rule_id": document.dataset_process_rule_id,
+ "dataset_process_rule": dataset_process_rules,
+ "document_process_rule": document_process_rules,
+ "name": document.name,
+ "created_from": document.created_from,
+ "created_by": document.created_by,
+ "created_at": int(document.created_at.timestamp()),
+ "tokens": document.tokens,
+ "indexing_status": document.indexing_status,
+ "completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,
+ "updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,
+ "indexing_latency": document.indexing_latency,
+ "error": document.error,
+ "enabled": document.enabled,
+ "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,
+ "disabled_by": document.disabled_by,
+ "archived": document.archived,
+ "doc_type": document.doc_type,
+ "doc_metadata": document.get_doc_metadata_details(session=session),
+ "segment_count": segment_count,
+ "average_segment_length": (document.word_count or 0) // segment_count if segment_count else 0,
+ "hit_count": document.get_hit_count(session=session),
+ "display_status": document.display_status,
+ "doc_form": document.doc_form,
+ "doc_language": document.doc_language,
+ "need_summary": document.need_summary if document.need_summary is not None else False,
+ }
exclude = metadata_fields if metadata == "without" else None
- return response.model_dump(mode="json", exclude=exclude, exclude_unset=True), 200
+ return dump_response(DocumentDetailResponse, response_payload, exclude=exclude, exclude_unset=True), 200
@setup_required
@login_required
diff --git a/api/libs/helper.py b/api/libs/helper.py
index 98a593cbe34..256e60ef500 100644
--- a/api/libs/helper.py
+++ b/api/libs/helper.py
@@ -208,9 +208,13 @@ def to_timestamp(value: datetime | int | None) -> int | None:
return value
-def dump_response(model: type[BaseModel], data: Any) -> dict[str, Any]:
- """Serialize a Pydantic response model to JSON-compatible dict output."""
- return model.model_validate(data, from_attributes=True).model_dump(mode="json")
+def dump_response(model: type[BaseModel], data: Any, **dump_kwargs: Any) -> dict[str, Any]:
+ """Serialize a Pydantic response model to JSON-compatible dict output.
+
+ Extra keyword arguments are forwarded to ``model_dump`` (e.g. ``include=...``,
+ ``exclude=...``, ``exclude_unset=True``).
+ """
+ return model.model_validate(data, from_attributes=True).model_dump(mode="json", **dump_kwargs)
def current_timestamp() -> int:
diff --git a/api/services/rag_pipeline/rag_pipeline_manage_service.py b/api/services/rag_pipeline/rag_pipeline_manage_service.py
index eaf797760fe..c01140ff934 100644
--- a/api/services/rag_pipeline/rag_pipeline_manage_service.py
+++ b/api/services/rag_pipeline/rag_pipeline_manage_service.py
@@ -9,7 +9,9 @@ logger = logging.getLogger(__name__)
class RagPipelineManageService:
@staticmethod
- def list_rag_pipeline_datasources(tenant_id: str) -> list[PluginDatasourceProviderEntity]:
+ def list_rag_pipeline_datasources(
+ tenant_id: str,
+ ) -> list[PluginDatasourceProviderEntity]:
"""
list rag pipeline datasources
"""
@@ -18,10 +20,16 @@ class RagPipelineManageService:
manager = PluginDatasourceManager()
datasources = manager.fetch_datasource_providers(tenant_id)
for datasource in datasources:
+ if not datasource.declaration.credentials_schema and not datasource.declaration.oauth_schema:
+ # built-in providers that declare neither credentials nor OAuth never require authorization
+ datasource.is_authorized = True
+ continue
datasource_provider_service = DatasourceProviderService()
try:
credentials = datasource_provider_service.get_datasource_credentials(
- tenant_id=tenant_id, provider=datasource.provider, plugin_id=datasource.plugin_id
+ tenant_id=tenant_id,
+ provider=datasource.provider,
+ plugin_id=datasource.plugin_id,
)
if credentials:
datasource.is_authorized = True
diff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_manage_service.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_manage_service.py
index faf93f0ce8e..dbe3955033f 100644
--- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_manage_service.py
+++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_manage_service.py
@@ -5,12 +5,33 @@ from pytest_mock import MockerFixture
from services.rag_pipeline.rag_pipeline_manage_service import RagPipelineManageService
+def _datasource(
+ provider: str,
+ plugin_id: str,
+ *,
+ credentials_schema: list[object] | None = None,
+ oauth_schema: object | None = None,
+) -> SimpleNamespace:
+ return SimpleNamespace(
+ provider=provider,
+ plugin_id=plugin_id,
+ is_authorized=False,
+ declaration=SimpleNamespace(
+ credentials_schema=credentials_schema if credentials_schema is not None else [object()],
+ oauth_schema=oauth_schema,
+ ),
+ )
+
+
def test_list_rag_pipeline_datasources_marks_authorized(mocker: MockerFixture) -> None:
- datasource_1 = SimpleNamespace(provider="notion", plugin_id="plugin-1", is_authorized=False)
- datasource_2 = SimpleNamespace(provider="jina", plugin_id="plugin-2", is_authorized=False)
+ datasource_1 = _datasource("notion", "plugin-1")
+ datasource_2 = _datasource("jina", "plugin-2")
manager_cls = mocker.patch("services.rag_pipeline.rag_pipeline_manage_service.PluginDatasourceManager")
- manager_cls.return_value.fetch_datasource_providers.return_value = [datasource_1, datasource_2]
+ manager_cls.return_value.fetch_datasource_providers.return_value = [
+ datasource_1,
+ datasource_2,
+ ]
provider_cls = mocker.patch("services.rag_pipeline.rag_pipeline_manage_service.DatasourceProviderService")
provider_instance = provider_cls.return_value
@@ -24,3 +45,51 @@ def test_list_rag_pipeline_datasources_marks_authorized(mocker: MockerFixture) -
assert result == [datasource_1, datasource_2]
assert datasource_1.is_authorized is True
assert datasource_2.is_authorized is False
+
+
+def test_list_rag_pipeline_datasources_marks_credential_free_providers_authorized(
+ mocker: MockerFixture,
+) -> None:
+ local_file = _datasource("local_file", "plugin-0", credentials_schema=[])
+ notion = _datasource("notion", "plugin-1")
+
+ manager_cls = mocker.patch("services.rag_pipeline.rag_pipeline_manage_service.PluginDatasourceManager")
+ manager_cls.return_value.fetch_datasource_providers.return_value = [
+ local_file,
+ notion,
+ ]
+
+ provider_cls = mocker.patch("services.rag_pipeline.rag_pipeline_manage_service.DatasourceProviderService")
+ provider_instance = provider_cls.return_value
+ provider_instance.get_datasource_credentials.return_value = None
+
+ result = RagPipelineManageService.list_rag_pipeline_datasources("tenant-1")
+
+ assert result == [local_file, notion]
+ assert local_file.is_authorized is True
+ assert notion.is_authorized is False
+ provider_instance.get_datasource_credentials.assert_called_once_with(
+ tenant_id="tenant-1", provider="notion", plugin_id="plugin-1"
+ )
+
+
+def test_oauth_only_provider_still_requires_authorization(
+ mocker: MockerFixture,
+) -> None:
+ oauth_only = _datasource("notion_oauth", "plugin-2", credentials_schema=[], oauth_schema=object())
+
+ manager_cls = mocker.patch("services.rag_pipeline.rag_pipeline_manage_service.PluginDatasourceManager")
+ manager_cls.return_value.fetch_datasource_providers.return_value = [oauth_only]
+
+ provider_cls = mocker.patch("services.rag_pipeline.rag_pipeline_manage_service.DatasourceProviderService")
+ provider_instance = provider_cls.return_value
+ provider_instance.get_datasource_credentials.return_value = None
+
+ result = RagPipelineManageService.list_rag_pipeline_datasources("tenant-1")
+
+ assert result == [oauth_only]
+ assert oauth_only.is_authorized is False
+ # the guard must not skip the credential lookup for OAuth-only providers
+ provider_instance.get_datasource_credentials.assert_called_once_with(
+ tenant_id="tenant-1", provider="notion_oauth", plugin_id="plugin-2"
+ )