Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(rag-pipeline): mark credential-free datasources as authorized (#39772)
ccl125d5a8ace1c4c1
diff --git a/api/controllers/console/app/workflow_run.py b/api/controllers/console/app/workflow_run.pyindex bc842dbf6b4..fb1a0361ce8 100644--- a/api/controllers/console/app/workflow_run.py+++ b/api/controllers/console/app/workflow_run.py@@ -37,7 +37,7 @@ from fields.workflow_run_fields import (from graphon.enums import WorkflowExecutionStatusfrom libs.archive_storage import ArchiveStorageNotConfiguredError, get_archive_storagefrom libs.custom_inputs import time_duration-from libs.helper import uuid_value+from libs.helper import dump_response, uuid_valuefrom libs.login import login_requiredfrom models import Account, App, AppMode, WorkflowArchiveLog, WorkflowRunTriggeredFromfrom models.workflow import WorkflowRun@@ -232,14 +232,17 @@ class WorkflowRunExportApi(Resource):expires_in=EXPORT_SIGNED_URL_EXPIRE_SECONDS,)expires_at = datetime.now(UTC) + timedelta(seconds=EXPORT_SIGNED_URL_EXPIRE_SECONDS)- response = WorkflowRunExportResponse.model_validate(- {- "status": "success",- "presigned_url": presigned_url,- "presigned_url_expires_at": expires_at.isoformat(),- }+ return (+ dump_response(+ WorkflowRunExportResponse,+ {+ "status": "success",+ "presigned_url": presigned_url,+ "presigned_url_expires_at": expires_at.isoformat(),+ },+ ),+ 200,)- return response.model_dump(mode="json"), 200@console_ns.route("/apps/<uuid:app_id>/advanced-chat/workflow-runs/count")diff --git a/api/controllers/console/datasets/datasets_document.py b/api/controllers/console/datasets/datasets_document.pyindex edb74bf50cc..44af59123d1 100644--- a/api/controllers/console/datasets/datasets_document.py+++ b/api/controllers/console/datasets/datasets_document.py@@ -1073,56 +1073,59 @@ class DocumentApi(DocumentResource):metadata_fields = {"doc_type", "doc_metadata"}if metadata == "only":- response = DocumentDetailResponse.model_validate(- {- "id": document.id,- "doc_type": document.doc_type,- "doc_metadata": document.get_doc_metadata_details(session=session),- }+ return (+ dump_response(+ DocumentDetailResponse,+ {+ "id": document.id,+ "doc_type": document.doc_type,+ "doc_metadata": document.get_doc_metadata_details(session=session),+ },+ include={"id", *metadata_fields},+ exclude_unset=True,+ ),+ 200,)- return response.model_dump(mode="json", include={"id", *metadata_fields}, exclude_unset=True), 200dataset_process_rules = DatasetService.get_process_rules(dataset_id_str, session)document_process_rule = document.get_dataset_process_rule(session=session)document_process_rules: Mapping[str, Any] = document_process_rule.to_dict() if document_process_rule else {}segment_count = document.get_segment_count(session=session)- response = DocumentDetailResponse.model_validate(- {- "id": document.id,- "position": document.position,- "data_source_type": document.data_source_type,- "data_source_info": document.data_source_info_dict,- "data_source_detail_dict": document.get_data_source_detail_dict(session=session),- "dataset_process_rule_id": document.dataset_process_rule_id,- "dataset_process_rule": dataset_process_rules,- "document_process_rule": document_process_rules,- "name": document.name,- "created_from": document.created_from,- "created_by": document.created_by,- "created_at": int(document.created_at.timestamp()),- "tokens": document.tokens,- "indexing_status": document.indexing_status,- "completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,- "updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,- "indexing_latency": document.indexing_latency,- "error": document.error,- "enabled": document.enabled,- "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,- "disabled_by": document.disabled_by,- "archived": document.archived,- "doc_type": document.doc_type,- "doc_metadata": document.get_doc_metadata_details(session=session),- "segment_count": segment_count,- "average_segment_length": (document.word_count or 0) // segment_count if segment_count else 0,- "hit_count": document.get_hit_count(session=session),- "display_status": document.display_status,- "doc_form": document.doc_form,- "doc_language": document.doc_language,- "need_summary": document.need_summary if document.need_summary is not None else False,- }- )+ response_payload = {+ "id": document.id,+ "position": document.position,+ "data_source_type": document.data_source_type,+ "data_source_info": document.data_source_info_dict,+ "data_source_detail_dict": document.get_data_source_detail_dict(session=session),+ "dataset_process_rule_id": document.dataset_process_rule_id,+ "dataset_process_rule": dataset_process_rules,+ "document_process_rule": document_process_rules,+ "name": document.name,+ "created_from": document.created_from,+ "created_by": document.created_by,+ "created_at": int(document.created_at.timestamp()),+ "tokens": document.tokens,+ "indexing_status": document.indexing_status,+ "completed_at": int(document.completed_at.timestamp()) if document.completed_at else None,+ "updated_at": int(document.updated_at.timestamp()) if document.updated_at else None,+ "indexing_latency": document.indexing_latency,+ "error": document.error,+ "enabled": document.enabled,+ "disabled_at": int(document.disabled_at.timestamp()) if document.disabled_at else None,+ "disabled_by": document.disabled_by,+ "archived": document.archived,+ "doc_type": document.doc_type,+ "doc_metadata": document.get_doc_metadata_details(session=session),+ "segment_count": segment_count,+ "average_segment_length": (document.word_count or 0) // segment_count if segment_count else 0,+ "hit_count": document.get_hit_count(session=session),+ "display_status": document.display_status,+ "doc_form": document.doc_form,+ "doc_language": document.doc_language,+ "need_summary": document.need_summary if document.need_summary is not None else False,+ }exclude = metadata_fields if metadata == "without" else None- return response.model_dump(mode="json", exclude=exclude, exclude_unset=True), 200+ return dump_response(DocumentDetailResponse, response_payload, exclude=exclude, exclude_unset=True), 200@setup_required@login_requireddiff --git a/api/libs/helper.py b/api/libs/helper.pyindex 98a593cbe34..256e60ef500 100644--- a/api/libs/helper.py+++ b/api/libs/helper.py@@ -208,9 +208,13 @@ def to_timestamp(value: datetime | int | None) -> int | None:return value-def dump_response(model: type[BaseModel], data: Any) -> dict[str, Any]:- """Serialize a Pydantic response model to JSON-compatible dict output."""- return model.model_validate(data, from_attributes=True).model_dump(mode="json")+def dump_response(model: type[BaseModel], data: Any, **dump_kwargs: Any) -> dict[str, Any]:+ """Serialize a Pydantic response model to JSON-compatible dict output.++ Extra keyword arguments are forwarded to ``model_dump`` (e.g. ``include=...``,+ ``exclude=...``, ``exclude_unset=True``).+ """+ return model.model_validate(data, from_attributes=True).model_dump(mode="json", **dump_kwargs)def current_timestamp() -> int:diff --git a/api/services/rag_pipeline/rag_pipeline_manage_service.py b/api/services/rag_pipeline/rag_pipeline_manage_service.pyindex eaf797760fe..c01140ff934 100644--- a/api/services/rag_pipeline/rag_pipeline_manage_service.py+++ b/api/services/rag_pipeline/rag_pipeline_manage_service.py@@ -9,7 +9,9 @@ logger = logging.getLogger(__name__)class RagPipelineManageService:@staticmethod- def list_rag_pipeline_datasources(tenant_id: str) -> list[PluginDatasourceProviderEntity]:+ def list_rag_pipeline_datasources(+ tenant_id: str,+ ) -> list[PluginDatasourceProviderEntity]:"""list rag pipeline datasources"""@@ -18,10 +20,16 @@ class RagPipelineManageService:manager = PluginDatasourceManager()datasources = manager.fetch_datasource_providers(tenant_id)for datasource in datasources:+ if not datasource.declaration.credentials_schema and not datasource.declaration.oauth_schema:+ # built-in providers that declare neither credentials nor OAuth never require authorization+ datasource.is_authorized = True+ continuedatasource_provider_service = DatasourceProviderService()try:credentials = datasource_provider_service.get_datasource_credentials(- tenant_id=tenant_id, provider=datasource.provider, plugin_id=datasource.plugin_id+ tenant_id=tenant_id,+ provider=datasource.provider,+ plugin_id=datasource.plugin_id,)if credentials:datasource.is_authorized = Truediff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_manage_service.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_manage_service.pyindex faf93f0ce8e..dbe3955033f 100644--- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_manage_service.py+++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_manage_service.py@@ -5,12 +5,33 @@ from pytest_mock import MockerFixturefrom services.rag_pipeline.rag_pipeline_manage_service import RagPipelineManageService+def _datasource(+ provider: str,+ plugin_id: str,+ *,+ credentials_schema: list[object] | None = None,+ oauth_schema: object | None = None,+) -> SimpleNamespace:+ return SimpleNamespace(+ provider=provider,+ plugin_id=plugin_id,+ is_authorized=False,+ declaration=SimpleNamespace(+ credentials_schema=credentials_schema if credentials_schema is not None else [object()],+ oauth_schema=oauth_schema,+ ),+ )++def test_list_rag_pipeline_datasources_marks_authorized(mocker: MockerFixture) -> None:- datasource_1 = SimpleNamespace(provider="notion", plugin_id="plugin-1", is_authorized=False)- datasource_2 = SimpleNamespace(provider="jina", plugin_id="plugin-2", is_authorized=False)+ datasource_1 = _datasource("notion", "plugin-1")+ datasource_2 = _datasource("jina", "plugin-2")manager_cls = mocker.patch("services.rag_pipeline.rag_pipeline_manage_service.PluginDatasourceManager")- manager_cls.return_value.fetch_datasource_providers.return_value = [datasource_1, datasource_2]+ manager_cls.return_value.fetch_datasource_providers.return_value = [+ datasource_1,+ datasource_2,+ ]provider_cls = mocker.patch("services.rag_pipeline.rag_pipeline_manage_service.DatasourceProviderService")provider_instance = provider_cls.return_value@@ -24,3 +45,51 @@ def test_list_rag_pipeline_datasources_marks_authorized(mocker: MockerFixture) -assert result == [datasource_1, datasource_2]assert datasource_1.is_authorized is Trueassert datasource_2.is_authorized is False+++def test_list_rag_pipeline_datasources_marks_credential_free_providers_authorized(+ mocker: MockerFixture,+) -> None:+ local_file = _datasource("local_file", "plugin-0", credentials_schema=[])+ notion = _datasource("notion", "plugin-1")++ manager_cls = mocker.patch("services.rag_pipeline.rag_pipeline_manage_service.PluginDatasourceManager")+ manager_cls.return_value.fetch_datasource_providers.return_value = [+ local_file,+ notion,+ ]++ provider_cls = mocker.patch("services.rag_pipeline.rag_pipeline_manage_service.DatasourceProviderService")+ provider_instance = provider_cls.return_value+ provider_instance.get_datasource_credentials.return_value = None++ result = RagPipelineManageService.list_rag_pipeline_datasources("tenant-1")++ assert result == [local_file, notion]+ assert local_file.is_authorized is True+ assert notion.is_authorized is False+ provider_instance.get_datasource_credentials.assert_called_once_with(+ tenant_id="tenant-1", provider="notion", plugin_id="plugin-1"+ )+++def test_oauth_only_provider_still_requires_authorization(+ mocker: MockerFixture,+) -> None:+ oauth_only = _datasource("notion_oauth", "plugin-2", credentials_schema=[], oauth_schema=object())++ manager_cls = mocker.patch("services.rag_pipeline.rag_pipeline_manage_service.PluginDatasourceManager")+ manager_cls.return_value.fetch_datasource_providers.return_value = [oauth_only]++ provider_cls = mocker.patch("services.rag_pipeline.rag_pipeline_manage_service.DatasourceProviderService")+ provider_instance = provider_cls.return_value+ provider_instance.get_datasource_credentials.return_value = None++ result = RagPipelineManageService.list_rag_pipeline_datasources("tenant-1")++ assert result == [oauth_only]+ assert oauth_only.is_authorized is False+ # the guard must not skip the credential lookup for OAuth-only providers+ provider_instance.get_datasource_credentials.assert_called_once_with(+ tenant_id="tenant-1", provider="notion_oauth", plugin_id="plugin-2"+ )