Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(oauth): serialize and persist provider credential refreshes (#32195)
Peter Pan5907db83b0d0
diff --git a/.github/workflows/knowledge-fs-ci.yml b/.github/workflows/knowledge-fs-ci.ymlindex 6883917c087..0b2757f47a8 100644--- a/.github/workflows/knowledge-fs-ci.yml+++ b/.github/workflows/knowledge-fs-ci.yml@@ -57,6 +57,7 @@ jobs:- 'api/.env.example'- 'api/app_factory.py'- 'api/commands/__init__.py'+ - 'api/controllers/common/errors.py'- 'api/controllers/console/__init__.py'- 'api/controllers/console/workspace/rbac/**'- 'api/controllers/inner_api/__init__.py'@@ -84,7 +85,10 @@ jobs:- 'api/tests/unit_tests/dev/test_generate_knowledge_fs_contract.py'- 'api/tests/unit_tests/extensions/test_ext_celery.py'- 'api/tests/unit_tests/models/test_agent_config_entities.py'+ - 'api/tests/unit_tests/controllers/common/test_errors.py'+ - 'api/tests/unit_tests/controllers/console/datasets/test_data_source.py'- 'api/tests/unit_tests/controllers/console/workspace/rbac/test_roles.py'+ - 'api/tests/unit_tests/controllers/inner_api/agent/test_tools.py'- 'api/tests/unit_tests/core/agent/test_base_agent_runner.py'- 'api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py'- 'api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py'@@ -196,6 +200,7 @@ jobs:production_touchpoints=(api/app_factory.pyapi/commands/__init__.py+ api/controllers/common/errors.pyapi/controllers/console/__init__.pyapi/controllers/console/datasets/datasets.pyapi/controllers/console/workspace/rbac/__init__.py@@ -257,9 +262,12 @@ jobs:test_touchpoints=(api/tests/unit_tests/extensions/test_ext_celery.pyapi/tests/unit_tests/models/test_agent_config_entities.py+ api/tests/unit_tests/controllers/common/test_errors.py+ api/tests/unit_tests/controllers/console/datasets/test_data_source.pyapi/tests/unit_tests/controllers/console/datasets/test_datasets.pyapi/tests/unit_tests/controllers/console/test_wraps.pyapi/tests/unit_tests/controllers/console/workspace/rbac/test_roles.py+ api/tests/unit_tests/controllers/inner_api/agent/test_tools.pyapi/tests/unit_tests/core/agent/test_base_agent_runner.pyapi/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.pyapi/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.pydiff --git a/api/.importlinter b/api/.importlinterindex 6f7a91ab446..789f8fec427 100644--- a/api/.importlinter+++ b/api/.importlinter@@ -313,6 +313,35 @@ forbidden_modules =flaskallow_indirect_imports = True+[importlinter:contract:tool-credential-application-boundary]+name = Runtime credential refresh policies are framework and persistence neutral+type = forbidden+source_modules =+ services.credentials.tool_runtime+ services.credentials.refresh+forbidden_modules =+ configs+ controllers+ core+ extensions+ flask+ models+ repositories+ sqlalchemy+ tasks+ werkzeug++[importlinter:contract:tool-credential-adapter-boundary]+name = Tool runtime credential persistence is reached only through composition+type = forbidden+source_modules =+ core.tools.tool_manager+forbidden_modules =+ repositories+ services.credentials.tool_runtime_adapters+ services.credentials.redis_refresh_lock+allow_indirect_imports = True+[importlinter:contract:credential-query-implementation-boundary]name = Credential query implementations are wired only by composition or other repositoriestype = forbiddendiff --git a/api/controllers/common/errors.py b/api/controllers/common/errors.pyindex 987023931ae..a8fdaf9810b 100644--- a/api/controllers/common/errors.py+++ b/api/controllers/common/errors.py@@ -7,6 +7,7 @@ from flask import current_app, got_request_exceptionfrom core.logging.context import get_request_idfrom libs.exception import BaseHTTPExceptionfrom libs.external_api import ExternalApi+from services.credentials.refresh import CredentialRefreshUnavailableErrorfrom services.errors.base import NoPermissionError@@ -181,3 +182,20 @@ def register_permission_error_handler(api: ExternalApi) -> None:# Flask-RESTX picks the first matching handler, including the generic Exception handler.cast(OrderedDict[type[Exception], object], api.error_handlers).move_to_end(NoPermissionError, last=False)api.document_error_handler(NoPermissionError, status=HTTPStatus.BAD_REQUEST, code=InvalidArgumentError.error_code)+++def register_credential_refresh_error_handler(api: ExternalApi) -> None:+ """Give temporary refresh failures the same retryable response on each API surface."""++ @api.errorhandler(CredentialRefreshUnavailableError)+ def handle_refresh_unavailable(error: CredentialRefreshUnavailableError) -> tuple[dict[str, str | int], int]:+ got_request_exception.send(current_app, exception=error)+ status = HTTPStatus.SERVICE_UNAVAILABLE+ return {"code": "service_unavailable", "message": str(error), "status": status.value}, status.value++ handlers: object = api.error_handlers+ assert isinstance(handlers, OrderedDict)+ handlers.move_to_end(CredentialRefreshUnavailableError, last=False)+ api.document_error_handler(+ CredentialRefreshUnavailableError, status=HTTPStatus.SERVICE_UNAVAILABLE, code="service_unavailable"+ )diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.pyindex 9af9886a7be..96253ec6c6f 100644--- a/api/controllers/console/__init__.py+++ b/api/controllers/console/__init__.py@@ -13,6 +13,7 @@ from controllers.common.errors import (InvalidArgumentError,NotFoundError,UnauthorizedError,+ register_credential_refresh_error_handler,register_permission_error_handler,)from controllers.console.error import NotInitValidateError, NotSetupError, UnauthorizedAndForceLogout@@ -36,6 +37,7 @@ api = ExternalApi(description="Console management APIs for app configuration, monitoring, and administration",)register_permission_error_handler(api)+register_credential_refresh_error_handler(api)# Errors the Console client handles centrally (bootstrap redirects, forced logout, malformed# input); any operation may answer with them, so no operation lists them. The shared# ``not_found``, ``unauthorized`` and ``internal_server_error`` codes are the ones Werkzeug's owndiff --git a/api/controllers/inner_api/__init__.py b/api/controllers/inner_api/__init__.pyindex 322066d6adf..d06c8be588c 100644--- a/api/controllers/inner_api/__init__.py+++ b/api/controllers/inner_api/__init__.py@@ -3,6 +3,7 @@ from types import ModuleTypefrom flask import Blueprintfrom flask_restx import Namespace+from controllers.common.errors import register_credential_refresh_error_handlerfrom libs.external_api import ExternalApibp = Blueprint("inner_api", __name__, url_prefix="/inner/api")@@ -22,6 +23,7 @@ api = ExternalApi(},security="InnerApiKey",)+register_credential_refresh_error_handler(api)# Create namespaceinner_api_ns = Namespace("inner_api", description="Internal API operations", path="/")diff --git a/api/core/helper/provider_cache.py b/api/core/helper/provider_cache.pyindex 8e85d444d2c..d4ad7b2bb99 100644--- a/api/core/helper/provider_cache.py+++ b/api/core/helper/provider_cache.py@@ -59,8 +59,35 @@ class SingletonProviderCredentialsCache(ProviderCredentialsCache):class ToolProviderCredentialsCache(ProviderCredentialsCache):"""Cache for tool provider credentials"""- def __init__(self, tenant_id: str, provider: str, credential_id: str):+ def __init__(self, tenant_id: str, provider: str, credential_id: str, *, fingerprint: str | None = None):+ """A fingerprint selects one snapshot-checked entry; None retains the legacy cache."""super().__init__(tenant_id=tenant_id, provider=provider, credential_id=credential_id)+ self._legacy_cache_key = self.cache_key+ self._fingerprint = fingerprint+ if fingerprint is not None:+ self.cache_key += ":snapshot"++ @override+ def get(self) -> dict[str, Any] | None:+ cached = super().get()+ if self._fingerprint is None or cached is None:+ return cached+ credentials = cached.get("credentials")+ if cached.get("fingerprint") == self._fingerprint and isinstance(credentials, dict):+ return credentials+ return None++ @override+ def set(self, config: dict[str, Any]) -> None:+ if self._fingerprint is not None:+ config = {"fingerprint": self._fingerprint, "credentials": config}+ super().set(config)++ @override+ def delete(self) -> None:+ # Separate commands also work when Redis Cluster puts the entries in different slots.+ redis_client.delete(self._legacy_cache_key)+ redis_client.delete(f"{self._legacy_cache_key}:snapshot")@overridedef _generate_cache_key(self, **kwargs: Any) -> str:diff --git a/api/core/tools/tool_manager.py b/api/core/tools/tool_manager.pyindex 20f4c5e67bd..b97a68ad914 100644--- a/api/core/tools/tool_manager.py+++ b/api/core/tools/tool_manager.py@@ -1,11 +1,9 @@-import jsonimport loggingimport mimetypes-import timefrom collections.abc import Generator, Mappingfrom os import listdir, pathfrom threading import Lock-from typing import TYPE_CHECKING, Any, Literal, Protocol, cast+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Protocol, castimport sqlalchemy as safrom pydantic import TypeAdapter@@ -18,10 +16,9 @@ import contextsfrom configs import dify_configfrom core.agent.entities import AgentToolEntityfrom core.app.entities.app_invoke_entities import InvokeFrom-from core.entities import PluginCredentialTypefrom core.helper.module_import_helper import load_single_subclass_from_sourcefrom core.helper.position_helper import is_filtered-from core.helper.provider_cache import ToolProviderCredentialsCache+from core.plugin.entities.plugin_daemon import CredentialTypefrom core.plugin.impl.exc import PluginDaemonNotFoundError, PluginNotFoundErrorfrom core.plugin.impl.tool import PluginToolManagerfrom core.tools.__base.tool import Tool@@ -43,14 +40,14 @@ from core.tools.entities.tool_entities import (ToolProviderType,emoji_icon_adapter,)-from core.tools.errors import ToolProviderCredentialValidationError, ToolProviderNotFoundError+from core.tools.errors import ToolProviderNotFoundErrorfrom core.tools.mcp_tool.provider import MCPToolProviderControllerfrom core.tools.mcp_tool.tool import MCPToolfrom core.tools.plugin_tool.provider import PluginToolProviderControllerfrom core.tools.plugin_tool.tool import PluginToolfrom core.tools.tool_label_manager import ToolLabelManagerfrom core.tools.utils.configuration import ToolParameterConfigurationManager-from core.tools.utils.encryption import create_provider_encrypter, create_tool_provider_encrypter+from core.tools.utils.encryption import create_tool_provider_encrypterfrom core.tools.utils.uuid_utils import is_valid_uuidfrom core.tools.workflow_as_tool.provider import WorkflowToolProviderControllerfrom core.tools.workflow_as_tool.tool import WorkflowTool@@ -93,12 +90,29 @@ class WorkflowToolRuntimeSpec(Protocol):def credential_id(self) -> str | None: ...+class ToolCredentialResolver(Protocol):+ def __call__(+ self,+ *,+ controller: BuiltinToolProviderController,+ provider_id: str,+ workspace_id: str,+ credential_id: str | None,+ ) -> tuple[Mapping[str, object], CredentialType]: ...++class ToolManager:_builtin_provider_lock = Lock()_hardcoded_providers: dict[str, BuiltinToolProviderController] = {}_external_hardcoded_provider_classes: list[type[BuiltinToolProviderController]] = []_builtin_providers_loaded = False_builtin_tools_labels: dict[str, I18nObject | None] = {}+ _credential_resolver: ClassVar[ToolCredentialResolver | None] = None++ @classmethod+ def configure_credential_resolver(cls, resolver: ToolCredentialResolver) -> None:+ """Register stateless composition; it must not capture an app, database or provider."""+ cls._credential_resolver = resolver@classmethoddef get_hardcoded_provider(cls, provider: str) -> BuiltinToolProviderController:@@ -229,141 +243,26 @@ class ToolManager:tool_invoke_from=tool_invoke_from,))- builtin_provider = None- if isinstance(provider_controller, PluginToolProviderController):- provider_id_entity = ToolProviderID(provider_id)- if is_valid_uuid(credential_id):- try:- builtin_provider_stmt = select(BuiltinToolProvider).where(- BuiltinToolProvider.tenant_id == tenant_id,- BuiltinToolProvider.id == credential_id,- )- builtin_provider = db.session.scalar(builtin_provider_stmt)- except Exception as e:- builtin_provider = None- logger.info("Error getting builtin provider %s:%s", credential_id, e, exc_info=True)- if builtin_provider is None:- raise ToolProviderCredentialValidationError(- f"Tool credential {credential_id} has been deleted. "- "Select or authorize another credential."- )-- if builtin_provider is None:- with Session(db.engine) as session:- builtin_provider = session.scalar(- sa.select(BuiltinToolProvider)- .where(- BuiltinToolProvider.tenant_id == tenant_id,- (BuiltinToolProvider.provider == str(provider_id_entity))- | (BuiltinToolProvider.provider == provider_id_entity.provider_name),- )- .order_by(BuiltinToolProvider.is_default.desc(), BuiltinToolProvider.created_at.asc())- )- if builtin_provider is None:- raise ToolProviderCredentialValidationError(- f"No workspace credential is configured for tool provider {provider_id}. "- "Authorize the provider or select a credential."- )- else:- builtin_provider = db.session.scalar(- select(BuiltinToolProvider)- .where(- BuiltinToolProvider.tenant_id == tenant_id, (BuiltinToolProvider.provider == provider_id)- )- .order_by(BuiltinToolProvider.is_default.desc(), BuiltinToolProvider.created_at.asc())- .limit(1)- )-- if builtin_provider is None:- raise ToolProviderCredentialValidationError(- f"No credential is configured for built-in tool provider {provider_id}. "- "Authorize the provider or select a credential."- )-- from core.helper.credential_utils import runtime_check_credential_policy_compliance-- runtime_check_credential_policy_compliance(- credential_id=builtin_provider.id,- provider=provider_id,- credential_type=PluginCredentialType.TOOL,- check_existence=False,+ selected_id = (+ credential_id+ if isinstance(provider_controller, PluginToolProviderController) and is_valid_uuid(credential_id)+ else None)-- encrypter, cache = create_provider_encrypter(- tenant_id=tenant_id,- config=[- x.to_basic_provider_config()- for x in provider_controller.get_credentials_schema_by_type(builtin_provider.credential_type)- ],- cache=ToolProviderCredentialsCache(- tenant_id=tenant_id, provider=provider_id, credential_id=builtin_provider.id- ),+ if cls._credential_resolver is None:+ raise RuntimeError("Tool credential resolver is not configured")+ credentials, credential_type = cls._credential_resolver(+ controller=provider_controller,+ provider_id=provider_id,+ workspace_id=tenant_id,+ credential_id=selected_id,)- decrypted_credentials: Mapping[str, Any] = encrypter.decrypt(builtin_provider.credentials)-- if builtin_provider.expires_at != -1 and (builtin_provider.expires_at - 60) < int(time.time()):- # TODO: circular import- from core.plugin.impl.oauth import OAuthHandler- from services.tools.builtin_tools_manage_service import BuiltinToolManageService-- tool_provider = ToolProviderID(provider_id)- provider_name = tool_provider.provider_name- redirect_uri = f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{provider_id}/tool/callback"- plugin_unique_identifier = (- provider_controller.plugin_unique_identifier- if isinstance(provider_controller, PluginToolProviderController)- else None- )- system_credentials = BuiltinToolManageService.get_oauth_client(- tenant_id,- provider_id,- provider_controller=(- provider_controller- if isinstance(provider_controller, PluginToolProviderController)- else None- ),- )- if system_credentials is not None:- plugin_unique_identifier = system_credentials.plugin_unique_identifier-- oauth_handler = OAuthHandler()- try:- refreshed_credentials = oauth_handler.refresh_credentials(- tenant_id=tenant_id,- user_id=builtin_provider.user_id,- plugin_id=tool_provider.plugin_id,- provider=provider_name,- redirect_uri=redirect_uri,- system_credentials=(- dict(system_credentials.credentials) if system_credentials is not None else {}- ),- plugin_unique_identifier=plugin_unique_identifier,- credentials=decrypted_credentials,- )- except Exception as exc:- logger.warning(- "Failed to refresh OAuth credentials for tool provider %s", provider_id, exc_info=True- )- raise ToolProviderCredentialValidationError(- f"OAuth credential for tool provider {provider_id} could not be refreshed. "- "Reauthorize or select another credential."- ) from exc- # update the credentials- builtin_provider.encrypted_credentials = json.dumps(- encrypter.encrypt(refreshed_credentials.credentials)- )- builtin_provider.expires_at = refreshed_credentials.expires_at- db.session.commit()- decrypted_credentials = refreshed_credentials.credentials- cache.delete()-return builtin_tool.fork_tool_runtime(runtime=ToolRuntime(tenant_id=tenant_id,user_id=user_id,- credentials=dict(decrypted_credentials),- credential_type=builtin_provider.credential_type,+ credentials=dict(credentials),+ credential_type=credential_type,runtime_parameters={},invoke_from=invoke_from,tool_invoke_from=tool_invoke_from,diff --git a/api/extensions/application_services/data_sources.py b/api/extensions/application_services/data_sources.pyindex 0818c635e47..a5a09a4ca7e 100644--- a/api/extensions/application_services/data_sources.py+++ b/api/extensions/application_services/data_sources.py@@ -10,11 +10,13 @@ from core.datasource.datasource_manager import DatasourceManagerfrom core.helper.ssrf_proxy import ssrf_proxyfrom core.plugin.impl.datasource import PluginDatasourceManagerfrom core.plugin.impl.oauth import OAuthHandler+from extensions.ext_redis import redis_clientfrom repositories.data_source.api_key_auth_repository import SQLAlchemyDataSourceApiKeyAuthBindingRepositoryfrom repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepositoryfrom repositories.data_source.oauth_binding_repository import SQLAlchemyDataSourceOAuthBindingRepositoryfrom repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepositoryfrom repositories.knowledge.document_repository import SQLAlchemyDocumentRepository+from services.credentials.redis_refresh_lock import RedisCredentialRefreshLockfrom services.data_source.auth.api_key_gateways import (ProviderApiKeyAuthCredentialValidator,TenantApiKeyAuthCredentialEncryptor,@@ -69,10 +71,21 @@ def build_data_source_credentials(*, database_client: sessionmaker[Session]) ->oauth_clients=PluginDatasourceOAuthClientResolver(configs=credentials, provider_manager=provider_manager),oauth_handler=OAuthHandler(),)+ refresh_lock = RedisCredentialRefreshLock(client=redis_client)return DataSourceCredentials(- providers=DatasourceProviderService(credentials=credentials),- actor=ActorAwareDatasourceCredentialGateway(credentials=credentials, codec=codec, refresher=refresher),- stored=TrustedStoredDatasourceCredentialGateway(credentials=credentials, codec=codec, refresher=refresher),+ providers=DatasourceProviderService(+ credentials=credentials,+ codec=codec,+ refresher=refresher,+ refresh_lock=refresh_lock,+ provider_manager=provider_manager,+ ),+ actor=ActorAwareDatasourceCredentialGateway(+ credentials=credentials, codec=codec, refresher=refresher, refresh_lock=refresh_lock+ ),+ stored=TrustedStoredDatasourceCredentialGateway(+ credentials=credentials, codec=codec, refresher=refresher, refresh_lock=refresh_lock+ ),)diff --git a/api/extensions/application_services/tools.py b/api/extensions/application_services/tools.pynew file mode 100644index 00000000000..67d36fc91a4--- /dev/null+++ b/api/extensions/application_services/tools.py@@ -0,0 +1,35 @@+"""Compose provider-scoped tool credential resolution for workflow execution."""++from collections.abc import Mapping++from configs import dify_config+from core.db.session_factory import get_session_maker+from core.plugin.entities.plugin_daemon import CredentialType+from core.tools.builtin_tool.provider import BuiltinToolProviderController+from core.tools.plugin_tool.provider import PluginToolProviderController+from extensions.ext_redis import redis_client+from models.provider_ids import ToolProviderID+from repositories.credentials.query_repository import CredentialQueryRepository+from services.credentials.redis_refresh_lock import RedisCredentialRefreshLock+from services.credentials.tool_runtime import ToolCredentialService+from services.credentials.tool_runtime_adapters import ProviderToolCredentialRuntime+++def resolve_tool_credentials(+ *, controller: BuiltinToolProviderController, provider_id: str, workspace_id: str, credential_id: str | None+) -> tuple[Mapping[str, object], CredentialType]:+ if isinstance(controller, PluginToolProviderController):+ provider = ToolProviderID(provider_id)+ providers = (str(provider), provider.provider_name)+ else:+ providers = (provider_id,)+ runtime = ProviderToolCredentialRuntime(+ controller=controller, provider_id=provider_id, console_api_url=dify_config.CONSOLE_API_URL+ )+ service = ToolCredentialService(+ credentials=CredentialQueryRepository(session_factory=get_session_maker()),+ runtime=runtime,+ refresh_lock=RedisCredentialRefreshLock(client=redis_client),+ providers=providers,+ )+ return runtime.resolve(service=service, workspace_id=workspace_id, credential_id=credential_id)diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.pyindex f55df0150ae..66d46a01180 100644--- a/api/extensions/ext_application_services.py+++ b/api/extensions/ext_application_services.py@@ -866,10 +866,12 @@ def build_application_services(def init_app(app: Flask) -> None:+ from extensions.application_services.tools import resolve_tool_credentialsfrom extensions.ext_login import bind_account_loaderapp.extensions["knowledge_fs_node_dependencies"] = build_knowledge_fs_node_dependenciesToolManager.register_hardcoded_provider_class(KnowledgeFSProvider)+ ToolManager.configure_credential_resolver(resolve_tool_credentials)services = build_application_services(database_client=get_session_maker(),deployment_edition=dify_config.DEPLOYMENT_EDITION,diff --git a/api/repositories/credentials/query_repository.py b/api/repositories/credentials/query_repository.pyindex 8e4d582738a..89605c4e85d 100644--- a/api/repositories/credentials/query_repository.py+++ b/api/repositories/credentials/query_repository.py@@ -3,9 +3,11 @@from collections.abc import Sequencefrom typing import override-from sqlalchemy import and_, func, or_, select+from sqlalchemy import and_, func, or_, select, update+from sqlalchemy.engine import CursorResultfrom sqlalchemy.orm import InstrumentedAttribute, Session, sessionmaker+from libs.datetime_utils import naive_utc_nowfrom models.credential_permission import CredentialPermission, CredentialTypefrom models.enums import PermissionEnumfrom models.oauth import DatasourceProvider@@ -20,6 +22,7 @@ from services.credentials.query import (ToolCredentialRecord,TriggerSubscriptionRecord,)+from services.credentials.tool_runtime import ToolRuntimeCredentialdef apply_credential_visibility_filter_for_actor(@@ -66,6 +69,55 @@ class CredentialQueryRepository(CredentialQuery):def __init__(self, *, session_factory: sessionmaker[Session]) -> None:self._session_factory = session_factory+ def get_tool_credential(+ self, *, workspace_id: str, providers: tuple[str, ...], credential_id: str | None+ ) -> ToolRuntimeCredential | None:+ statement = select(BuiltinToolProvider).where(+ BuiltinToolProvider.tenant_id == workspace_id,+ BuiltinToolProvider.provider.in_(providers),+ )+ if credential_id is not None:+ statement = statement.where(BuiltinToolProvider.id == credential_id)+ with self._session_factory() as session:+ record = session.scalar(+ statement.order_by(BuiltinToolProvider.is_default.desc(), BuiltinToolProvider.created_at.asc()).limit(1)+ )+ if record is None:+ return None+ return ToolRuntimeCredential(+ id=record.id,+ workspace_id=workspace_id,+ owner_id=record.user_id,+ provider=record.provider,+ credential_type=record.credential_type.value,+ encrypted_credentials=record.encrypted_credentials,+ expires_at=record.expires_at,+ )++ def update_tool_credential_if_unchanged(+ self, *, record: ToolRuntimeCredential, encrypted_credentials: str, expires_at: int+ ) -> bool:+ with self._session_factory.begin() as session:+ result = session.execute(+ update(BuiltinToolProvider)+ .where(+ BuiltinToolProvider.id == record.id,+ BuiltinToolProvider.tenant_id == record.workspace_id,+ BuiltinToolProvider.provider == record.provider,+ BuiltinToolProvider.user_id == record.owner_id,+ BuiltinToolProvider.credential_type == record.credential_type,+ BuiltinToolProvider.encrypted_credentials == record.encrypted_credentials,+ BuiltinToolProvider.expires_at == record.expires_at,+ )+ .values(+ encrypted_credentials=encrypted_credentials,+ expires_at=expires_at,+ updated_at=naive_utc_now(),+ )+ )+ assert isinstance(result, CursorResult)+ return result.rowcount == 1+@overridedef list_models(self, *, workspace_id: str, provider: str, actor_id: str | None) -> Sequence[ModelCredentialRecord]:names = [provider]diff --git a/api/repositories/data_source/credential_repository.py b/api/repositories/data_source/credential_repository.pyindex ae38825e118..6121cc3afb7 100644--- a/api/repositories/data_source/credential_repository.py+++ b/api/repositories/data_source/credential_repository.py@@ -256,6 +256,7 @@ def _update_datasource_credentials_if_unchanged(DatasourceProvider.provider == record.provider,DatasourceProvider.plugin_id == record.plugin_id,DatasourceProvider.user_id == record.owner_id,+ DatasourceProvider.auth_type == record.auth_type,credentials_match,DatasourceProvider.expires_at == record.expires_at,)diff --git a/api/services/agent_tool_inner_service.py b/api/services/agent_tool_inner_service.pyindex 213e9ace816..085f37d471c 100644--- a/api/services/agent_tool_inner_service.py+++ b/api/services/agent_tool_inner_service.py@@ -34,6 +34,7 @@ from core.tools.tool_engine import ToolEnginefrom core.tools.tool_manager import ToolManagerfrom core.tools.utils.message_transformer import ToolFileMessageTransformerfrom models.model import App+from services.credentials.refresh import CredentialRefreshUnavailableErrorfrom services.entities.agent_tool_inner import AgentToolInvokeRequest, AgentToolInvokeResponsefrom services.errors.agent_tool_inner import AgentToolInnerServiceErrorfrom services.workflow_run_agg import WorkflowRunAgg@@ -106,6 +107,8 @@ class AgentToolInnerService:conversation_id=request.caller.conversation_id,))+ except CredentialRefreshUnavailableError:+ raiseexcept ToolProviderNotFoundError as exc:raise AgentToolInnerServiceError(error_code="agent_tool_declaration_not_found",diff --git a/api/services/credentials/redis_refresh_lock.py b/api/services/credentials/redis_refresh_lock.pynew file mode 100644index 00000000000..627af38219e--- /dev/null+++ b/api/services/credentials/redis_refresh_lock.py@@ -0,0 +1,76 @@+"""Redis leases for OAuth refreshes that can outlive a fixed lock timeout."""++import logging+from collections.abc import Generator+from contextlib import contextmanager+from threading import Event, Thread+from typing import Literal++from redis.exceptions import LockError, RedisError+from redis.lock import Lock++from extensions.ext_redis import RedisClientWrapper+from services.credentials.refresh import CredentialRefreshLease, CredentialRefreshUnavailableError++logger = logging.getLogger(__name__)++_LEASE_SECONDS = 60+_WAIT_SECONDS = 30+_RENEW_SECONDS = 20+++class _RefreshLease:+ def __init__(self, lock: Lock, lost: Event) -> None:+ self._lock = lock+ self._lost = lost++ def ensure_owned(self) -> None:+ if self._lost.is_set():+ raise CredentialRefreshUnavailableError()+ try:+ self._lock.reacquire()+ except (LockError, RedisError) as exc:+ self._lost.set()+ raise CredentialRefreshUnavailableError() from exc+++class RedisCredentialRefreshLock:+ def __init__(self, *, client: RedisClientWrapper) -> None:+ self._client = client++ @contextmanager+ def acquire(+ self, *, credential_type: Literal["tool", "datasource"], workspace_id: str, credential_id: str+ ) -> Generator[CredentialRefreshLease]:+ name = f"oauth:credential-refresh:{credential_type}:{workspace_id}:{credential_id}"+ lock = self._client.lock(name, timeout=_LEASE_SECONDS, blocking_timeout=_WAIT_SECONDS, thread_local=False)+ try:+ if not lock.acquire():+ raise CredentialRefreshUnavailableError()+ except (LockError, RedisError) as exc:+ raise CredentialRefreshUnavailableError() from exc++ stopped, lost = Event(), Event()+ lease = _RefreshLease(lock, lost)++ def renew() -> None:+ while not stopped.wait(_RENEW_SECONDS):+ try:+ lease.ensure_owned()+ except Exception:+ lost.set()+ logger.warning("Lost credential refresh lease: %s", name, exc_info=True)+ return++ heartbeat = Thread(target=renew, daemon=True, name=f"CredentialRefresh({credential_id})")+ try:+ heartbeat.start()+ yield lease+ finally:+ stopped.set()+ if heartbeat.ident is not None:+ heartbeat.join(timeout=2)+ try:+ lock.release()+ except (LockError, RedisError):+ logger.warning("Failed to release credential refresh lease: %s", name, exc_info=True)diff --git a/api/services/credentials/refresh.py b/api/services/credentials/refresh.pynew file mode 100644index 00000000000..477b27baecc--- /dev/null+++ b/api/services/credentials/refresh.py@@ -0,0 +1,35 @@+"""Coordination contracts for refreshing one persisted provider credential."""++from contextlib import AbstractContextManager+from typing import Literal, Protocol+++def credential_needs_refresh(expires_at: int, *, now: float) -> bool:+ return expires_at != -1 and expires_at - 60 < int(now)+++class CredentialRefreshUnavailableError(RuntimeError):+ """Temporary coordination failure; preserve it until the caller chooses a retry response."""++ def __init__(self) -> None:+ super().__init__("Credential refresh is temporarily unavailable. Please retry.")+++class CredentialRefreshLease(Protocol):+ def ensure_owned(self) -> None:+ """Renew ownership or fail before issuing a refresh or persisting its result."""+ ...+++class CredentialRefreshLock(Protocol):+ def acquire(+ self, *, credential_type: Literal["tool", "datasource"], workspace_id: str, credential_id: str+ ) -> AbstractContextManager[CredentialRefreshLease]:+ """Serialize refreshes across workers, with bounded waiting and a renewed lease.++ Callers reload the credential after acquisition and keep this lease through+ the external refresh and the committed conditional write. No database+ transaction may span the external request. Failure never permits an+ unlocked refresh or a return of the original expired credential.+ """+ ...diff --git a/api/services/credentials/tool_runtime.py b/api/services/credentials/tool_runtime.pynew file mode 100644index 00000000000..15fb19cf99d--- /dev/null+++ b/api/services/credentials/tool_runtime.py@@ -0,0 +1,122 @@+"""Resolve tool credentials through scoped snapshots and serialized OAuth refreshes."""++import time+from collections.abc import Mapping+from dataclasses import dataclass+from typing import Protocol++from services.credentials.refresh import CredentialRefreshLock, credential_needs_refresh+++@dataclass(frozen=True, slots=True)+class ToolRuntimeCredential:+ """Detached provider state; absent ciphertext represents empty credentials."""++ id: str+ workspace_id: str+ owner_id: str+ provider: str+ credential_type: str+ encrypted_credentials: str | None+ expires_at: int+++@dataclass(frozen=True, slots=True)+class RefreshedToolCredential:+ credentials: Mapping[str, object]+ expires_at: int+++@dataclass(frozen=True, slots=True)+class ResolvedToolCredential:+ credentials: Mapping[str, object]+ credential_type: str+++class ToolCredentialNotFoundError(ValueError):+ pass+++class ToolCredentialRefreshError(ValueError):+ pass+++class ToolCredentialStore(Protocol):+ def get_tool_credential(+ self, *, workspace_id: str, providers: tuple[str, ...], credential_id: str | None+ ) -> ToolRuntimeCredential | None:+ """Return a detached snapshot, scoped to workspace and provider, preferring the default when ID is absent."""+ ...++ def update_tool_credential_if_unchanged(+ self, *, record: ToolRuntimeCredential, encrypted_credentials: str, expires_at: int+ ) -> bool:+ """Commit in a short transaction, guarding the owner, auth type, expiry and previous ciphertext."""+ ...+++class ToolCredentialRuntime(Protocol):+ def check_policy(self, record: ToolRuntimeCredential) -> None: ...++ def decrypt(self, record: ToolRuntimeCredential) -> dict[str, object]: ...++ def encrypt(self, record: ToolRuntimeCredential, credentials: Mapping[str, object]) -> str: ...++ def refresh(self, record: ToolRuntimeCredential, credentials: Mapping[str, object]) -> RefreshedToolCredential: ...++ def invalidate(self, record: ToolRuntimeCredential) -> None: ...+++class ToolCredentialService:+ def __init__(+ self,+ *,+ credentials: ToolCredentialStore,+ runtime: ToolCredentialRuntime,+ refresh_lock: CredentialRefreshLock,+ providers: tuple[str, ...],+ ) -> None:+ self._credentials = credentials+ self._runtime = runtime+ self._refresh_lock = refresh_lock+ self._providers = providers++ def resolve(self, *, workspace_id: str, credential_id: str | None) -> ResolvedToolCredential:+ record = self._load(workspace_id=workspace_id, credential_id=credential_id)+ self._runtime.check_policy(record)+ if not credential_needs_refresh(record.expires_at, now=time.time()):+ return self._resolved(record)++ with self._refresh_lock.acquire(+ credential_type="tool", workspace_id=workspace_id, credential_id=record.id+ ) as lease:+ record = self._load(workspace_id=workspace_id, credential_id=record.id)+ self._runtime.check_policy(record)+ if not credential_needs_refresh(record.expires_at, now=time.time()):+ return self._resolved(record)+ decrypted = self._runtime.decrypt(record)+ lease.ensure_owned()+ refreshed = self._runtime.refresh(record, decrypted)+ encrypted = self._runtime.encrypt(record, refreshed.credentials)+ lease.ensure_owned()+ if not self._credentials.update_tool_credential_if_unchanged(+ record=record, encrypted_credentials=encrypted, expires_at=refreshed.expires_at+ ):+ latest = self._load(workspace_id=workspace_id, credential_id=record.id)+ self._runtime.check_policy(latest)+ if credential_needs_refresh(latest.expires_at, now=time.time()):+ raise ToolCredentialRefreshError("Credential changed while it was being refreshed. Please retry.")+ return self._resolved(latest)+ self._runtime.invalidate(record)+ return ResolvedToolCredential(refreshed.credentials, record.credential_type)++ def _load(self, *, workspace_id: str, credential_id: str | None) -> ToolRuntimeCredential:+ record = self._credentials.get_tool_credential(+ workspace_id=workspace_id, providers=self._providers, credential_id=credential_id+ )+ if record is None:+ raise ToolCredentialNotFoundError("Tool credential not found")+ return record++ def _resolved(self, record: ToolRuntimeCredential) -> ResolvedToolCredential:+ return ResolvedToolCredential(self._runtime.decrypt(record), record.credential_type)diff --git a/api/services/credentials/tool_runtime_adapters.py b/api/services/credentials/tool_runtime_adapters.pynew file mode 100644index 00000000000..b452f1e95b4--- /dev/null+++ b/api/services/credentials/tool_runtime_adapters.py@@ -0,0 +1,126 @@+"""Provider policy, encryption and OAuth adapters for runtime tool credentials."""++import json+import logging+from collections.abc import Mapping+from hashlib import sha256++from core.entities import PluginCredentialType+from core.helper.credential_utils import runtime_check_credential_policy_compliance+from core.helper.provider_cache import ToolProviderCredentialsCache+from core.plugin.entities.plugin_daemon import CredentialType+from core.plugin.impl.oauth import OAuthHandler+from core.tools.builtin_tool.provider import BuiltinToolProviderController+from core.tools.errors import ToolProviderCredentialValidationError+from core.tools.plugin_tool.provider import PluginToolProviderController+from core.tools.utils.encryption import ProviderConfigEncrypter, create_provider_encrypter+from models.provider_ids import ToolProviderID+from services.credentials.tool_runtime import (+ RefreshedToolCredential,+ ToolCredentialNotFoundError,+ ToolCredentialRefreshError,+ ToolCredentialService,+ ToolRuntimeCredential,+)+from services.tools.builtin_tools_manage_service import BuiltinToolManageService++logger = logging.getLogger(__name__)+++class ProviderToolCredentialRuntime:+ def __init__(self, *, controller: BuiltinToolProviderController, provider_id: str, console_api_url: str) -> None:+ self._controller = controller+ self._provider_id = provider_id+ self._console_api_url = console_api_url++ def resolve(+ self, *, service: ToolCredentialService, workspace_id: str, credential_id: str | None+ ) -> tuple[Mapping[str, object], CredentialType]:+ """Map credential failures to tool errors, retaining temporary coordination failures."""+ try:+ resolved = service.resolve(workspace_id=workspace_id, credential_id=credential_id)+ except ToolCredentialNotFoundError as exc:+ if credential_id is not None:+ message = f"Tool credential {credential_id} has been deleted. Select or authorize another credential."+ elif isinstance(self._controller, PluginToolProviderController):+ message = (+ f"No workspace credential is configured for tool provider {self._provider_id}. "+ "Authorize the provider or select a credential."+ )+ else:+ message = (+ f"No credential is configured for built-in tool provider {self._provider_id}. "+ "Authorize the provider or select a credential."+ )+ raise ToolProviderCredentialValidationError(message) from exc+ except ToolCredentialRefreshError as exc:+ raise ToolProviderCredentialValidationError(str(exc)) from exc+ return resolved.credentials, CredentialType(resolved.credential_type)++ def check_policy(self, record: ToolRuntimeCredential) -> None:+ runtime_check_credential_policy_compliance(+ credential_id=record.id,+ provider=self._provider_id,+ credential_type=PluginCredentialType.TOOL,+ check_existence=False,+ )++ def _cache(self, record: ToolRuntimeCredential) -> ToolProviderCredentialsCache:+ return ToolProviderCredentialsCache(+ tenant_id=record.workspace_id,+ provider=self._provider_id,+ credential_id=record.id,+ fingerprint=(+ sha256((record.encrypted_credentials or "{}").encode()).hexdigest()+ if record.credential_type == "oauth2"+ else None+ ),+ )++ def _encrypter(self, record: ToolRuntimeCredential) -> ProviderConfigEncrypter:+ encrypter, _ = create_provider_encrypter(+ tenant_id=record.workspace_id,+ config=[+ item.to_basic_provider_config()+ for item in self._controller.get_credentials_schema_by_type(CredentialType(record.credential_type))+ ],+ cache=self._cache(record),+ )+ return encrypter++ def decrypt(self, record: ToolRuntimeCredential) -> dict[str, object]:+ return dict(self._encrypter(record).decrypt(json.loads(record.encrypted_credentials or "{}")))++ def encrypt(self, record: ToolRuntimeCredential, credentials: Mapping[str, object]) -> str:+ return json.dumps(self._encrypter(record).encrypt(credentials))++ def invalidate(self, record: ToolRuntimeCredential) -> None:+ self._cache(record).delete()++ def refresh(self, record: ToolRuntimeCredential, credentials: Mapping[str, object]) -> RefreshedToolCredential:+ provider = ToolProviderID(self._provider_id)+ controller = self._controller if isinstance(self._controller, PluginToolProviderController) else None+ client = BuiltinToolManageService.get_oauth_client(+ record.workspace_id, self._provider_id, provider_controller=controller+ )+ identifier = (+ client.plugin_unique_identifier if client else (controller.plugin_unique_identifier if controller else None)+ )+ try:+ refreshed = OAuthHandler().refresh_credentials(+ tenant_id=record.workspace_id,+ user_id=record.owner_id,+ plugin_id=provider.plugin_id,+ provider=provider.provider_name,+ redirect_uri=f"{self._console_api_url}/console/api/oauth/plugin/{self._provider_id}/tool/callback",+ system_credentials=dict(client.credentials) if client is not None else {},+ plugin_unique_identifier=identifier,+ credentials=credentials,+ )+ except Exception as exc:+ logger.warning("Failed to refresh OAuth credentials for tool provider %s", self._provider_id, exc_info=True)+ raise ToolCredentialRefreshError(+ f"OAuth credential for tool provider {self._provider_id} could not be refreshed. "+ "Reauthorize or select another credential."+ ) from exc+ return RefreshedToolCredential(refreshed.credentials, refreshed.expires_at)diff --git a/api/services/data_source/credential_gateway.py b/api/services/data_source/credential_gateway.pyindex 75dc5cf7cf4..8f4e1501140 100644--- a/api/services/data_source/credential_gateway.py+++ b/api/services/data_source/credential_gateway.py@@ -6,6 +6,11 @@ from collections.abc import Callable, Mapping, Sequencefrom dataclasses import dataclassfrom typing import Protocol+from services.credentials.refresh import (+ CredentialRefreshLock,+ CredentialRefreshUnavailableError,+ credential_needs_refresh,+)from services.data_source.entities.credential import DatasourceCredentialRecord, DatasourceOAuthClientConfigRecordlogger = logging.getLogger(__name__)@@ -27,6 +32,7 @@ def resolve_stored_notion_access_token(default must not change the integration used to read an existing document.Preserve indexing's environment fallback when credential resolution fails,including refresh failures. Actor-selected imports do not use this policy.+ Without a fallback, preserve temporary refresh failures so callers can retry."""credential_error: Exception | None = Noneif credential_id:@@ -38,6 +44,8 @@ def resolve_stored_notion_access_token(credential_error = errorif not integration_token:+ if isinstance(credential_error, CredentialRefreshUnavailableError):+ raise credential_errorraise NotionCredentialUnavailableError("Must specify `integration_token` or set environment variable `NOTION_INTEGRATION_TOKEN`.") from credential_error@@ -63,7 +71,9 @@ class DatasourceProviderCredentialStore(Protocol):def update_if_unchanged(self, *, record: DatasourceCredentialRecord, encrypted_credentials: Mapping[str, object], expires_at: int- ) -> bool: ...+ ) -> bool:+ """Commit a scoped write guarded by owner, auth type, ciphertext and expiry; return false on a conflict."""+ ...class DatasourceCredentialStore(Protocol):@@ -189,11 +199,13 @@ class ActorAwareDatasourceCredentialGateway:credentials: DatasourceCredentialStore,codec: DatasourceCredentialCodec,refresher: DatasourceCredentialRefresher,+ refresh_lock: CredentialRefreshLock,now: Callable[[], float] = time.time,) -> None:self._credentials = credentialsself._codec = codecself._refresher = refresher+ self._refresh_lock = refresh_lockself._now = nowdef resolve(@@ -224,6 +236,7 @@ class ActorAwareDatasourceCredentialGateway:refresher=self._refresher,update=self._credentials.update_if_unchanged,load_latest=load_record,+ refresh_lock=self._refresh_lock,now=self._now,)@@ -237,11 +250,13 @@ class TrustedStoredDatasourceCredentialGateway:credentials: TrustedDatasourceCredentialStore,codec: DatasourceCredentialCodec,refresher: DatasourceCredentialRefresher,+ refresh_lock: CredentialRefreshLock,now: Callable[[], float] = time.time,) -> None:self._credentials = credentialsself._codec = codecself._refresher = refresher+ self._refresh_lock = refresh_lockself._now = nowdef resolve_for_document(@@ -270,19 +285,16 @@ class TrustedStoredDatasourceCredentialGateway:record = load_record()return resolve_credential_snapshot(record,- actor_id=record.owner_id,+ actor_id=None,codec=self._codec,refresher=self._refresher,update=self._credentials.update_if_unchanged,load_latest=load_record,+ refresh_lock=self._refresh_lock,now=self._now,)-def credential_needs_refresh(expires_at: int, *, now: float) -> bool:- return expires_at != -1 and expires_at - 60 < int(now)--class CredentialSnapshotUpdate(Protocol):def __call__(self, *, record: DatasourceCredentialRecord, encrypted_credentials: Mapping[str, object], expires_at: int@@ -297,23 +309,38 @@ def resolve_credential_snapshot(refresher: DatasourceCredentialRefresher,update: CredentialSnapshotUpdate,load_latest: Callable[[], DatasourceCredentialRecord],+ refresh_lock: CredentialRefreshLock,now: Callable[[], float],) -> dict[str, object]:- decrypted = codec.decrypt(record)+ """Refresh under one lease; an absent actor selects the persisted credential owner.++ Every caller, including legacy provider execution, uses the same credential+ lock. Reloading also rechecks the caller's visibility or stored owner chain.+ """if not credential_needs_refresh(record.expires_at, now=now()):- return decrypted- if actor_id is None:- raise DatasourceCredentialRefreshError(record.id)- refreshed = refresher.refresh(- workspace_id=record.workspace_id, actor_id=actor_id, record=record, credentials=decrypted- )- if update(- record=record,- encrypted_credentials=codec.encrypt(record, refreshed.credentials),- expires_at=refreshed.expires_at,- ):- return dict(refreshed.credentials)- latest = load_latest()- if credential_needs_refresh(latest.expires_at, now=now()):- raise DatasourceCredentialConcurrentUpdateError()- return codec.decrypt(latest)+ return codec.decrypt(record)+ with refresh_lock.acquire(+ credential_type="datasource", workspace_id=record.workspace_id, credential_id=record.id+ ) as lease:+ latest = load_latest()+ if (latest.workspace_id, latest.id) != (record.workspace_id, record.id):+ raise DatasourceCredentialConcurrentUpdateError()+ record = latest+ if not credential_needs_refresh(record.expires_at, now=now()):+ return codec.decrypt(record)+ refresh_actor = actor_id if actor_id is not None else record.owner_id+ if refresh_actor is None:+ raise DatasourceCredentialRefreshError(record.id)+ decrypted = codec.decrypt(record)+ lease.ensure_owned()+ refreshed = refresher.refresh(+ workspace_id=record.workspace_id, actor_id=refresh_actor, record=record, credentials=decrypted+ )+ encrypted = codec.encrypt(record, refreshed.credentials)+ lease.ensure_owned()+ if update(record=record, encrypted_credentials=encrypted, expires_at=refreshed.expires_at):+ return dict(refreshed.credentials)+ latest = load_latest()+ if credential_needs_refresh(latest.expires_at, now=now()):+ raise DatasourceCredentialConcurrentUpdateError()+ return codec.decrypt(latest)diff --git a/api/services/data_source/provider_service.py b/api/services/data_source/provider_service.pyindex 3b8a73acb0d..8ca0f4fe829 100644--- a/api/services/data_source/provider_service.py+++ b/api/services/data_source/provider_service.py@@ -1,10 +1,13 @@import loggingimport timefrom collections.abc import Mapping-from dataclasses import replacefrom typing import TYPE_CHECKING, Any, castfrom services.credentials.query import CredentialQuery+from services.credentials.refresh import (+ CredentialRefreshLock,+ credential_needs_refresh,+)from services.plugin.dispatch_identity import PluginOAuthClientConfigif TYPE_CHECKING:@@ -20,7 +23,6 @@ from core.helper.name_generator import generate_incremental_namefrom core.helper.provider_cache import NoOpProviderCredentialCachefrom core.plugin.entities.plugin_daemon import CredentialType, PluginDatasourceProviderEntityfrom core.plugin.impl.datasource import PluginDatasourceManager-from core.plugin.impl.oauth import OAuthHandlerfrom core.tools.utils.encryption import ProviderConfigCache, ProviderConfigEncrypter, create_provider_encrypterfrom extensions.ext_database import dbfrom extensions.ext_redis import redis_client@@ -29,14 +31,16 @@ from models.oauth import DatasourceOauthParamConfig, DatasourceOauthTenantParamCfrom models.provider_ids import DatasourceProviderIDfrom services.data_source.credential_adapters import (datasource_secret_variables,- refresh_datasource_credential,resolve_datasource_oauth_client,transform_secret_fields,)from services.data_source.credential_gateway import (+ DatasourceCredentialCodec,DatasourceCredentialError,+ DatasourceCredentialNotFoundError,+ DatasourceCredentialRefresher,DatasourceProviderCredentialStore,- credential_needs_refresh,+ resolve_credential_snapshot,resolve_stored_notion_access_token,)from services.data_source.entities.credential import DatasourceCredentialRecord@@ -65,9 +69,20 @@ class DatasourceProviderService:Datasource provider credentials and authorization management."""- def __init__(self, *, credentials: DatasourceProviderCredentialStore) -> None:+ def __init__(+ self,+ *,+ credentials: DatasourceProviderCredentialStore,+ codec: DatasourceCredentialCodec,+ refresher: DatasourceCredentialRefresher,+ refresh_lock: CredentialRefreshLock,+ provider_manager: PluginDatasourceManager,+ ) -> None:self._credentials = credentials- self.provider_manager = PluginDatasourceManager()+ self._codec = codec+ self._refresher = refresher+ self._refresh_lock = refresh_lock+ self.provider_manager = provider_managerdef get_stored_notion_access_token(self, *, tenant_id: str, credential_id: str | None) -> str:"""Resolve a stored Notion credential with the configured integration fallback."""@@ -90,98 +105,6 @@ class DatasourceProviderService:integration_token=dify_config.NOTION_INTEGRATION_TOKEN,)- @staticmethod- def _should_refresh_credentials(- datasource_provider: DatasourceProvider | DatasourceCredentialRecord, now: int | None = None- ) -> bool:- current_time = int(time.time()) if now is None else now- return credential_needs_refresh(datasource_provider.expires_at, now=current_time)-- def _refresh_datasource_credentials(- self,- tenant_id: str,- provider: str,- plugin_id: str,- datasource_provider: DatasourceProvider | DatasourceCredentialRecord,- current_user: Any,- ) -> tuple[dict[str, Any], int]:- datasource_provider_id = DatasourceProviderID(f"{plugin_id}/{provider}")- provider_name = datasource_provider_id.provider_name- provider_declaration = self.provider_manager.fetch_datasource_provider(- tenant_id=tenant_id, provider_id=str(datasource_provider_id)- )- plugin_unique_identifier = provider_declaration.plugin_unique_identifier- credential_id = getattr(datasource_provider, "id", None)- credential_name = getattr(datasource_provider, "name", None)- logger.info(- "Refreshing datasource credentials for provider %s",- provider_name,- extra={- "tenant_id": tenant_id,- "plugin_id": datasource_provider_id.plugin_id,- "provider": provider_name,- "credential_id": credential_id,- "credential_name": credential_name,- "expires_at": datasource_provider.expires_at,- },- )- decrypted_credentials = self.decrypt_datasource_provider_credentials(- tenant_id=tenant_id,- datasource_provider=datasource_provider,- plugin_id=plugin_id,- provider=provider,- provider_declaration=provider_declaration,- )- system_credentials = self.get_oauth_client(- tenant_id, datasource_provider_id, provider_declaration=provider_declaration- )- try:- refreshed_credentials = refresh_datasource_credential(- OAuthHandler(),- workspace_id=tenant_id,- actor_id=current_user.id,- provider_id=datasource_provider_id,- system_credentials=system_credentials or PluginOAuthClientConfig(plugin_unique_identifier, {}),- credentials=decrypted_credentials,- )- except Exception as exc:- message = (- f"Failed to refresh datasource credentials for provider {provider_name}"- f" (credential: {credential_name or credential_id or 'unknown'})"- )- logger.exception(- message,- extra={- "tenant_id": tenant_id,- "plugin_id": datasource_provider_id.plugin_id,- "provider": provider_name,- "credential_id": credential_id,- "credential_name": credential_name,- },- )- raise ValueError(f"{message}: {exc}") from exc- encrypted_credentials = self.encrypt_datasource_provider_credentials(- tenant_id=tenant_id,- raw_credentials=refreshed_credentials.credentials,- provider=provider,- provider_declaration=provider_declaration,- plugin_id=plugin_id,- datasource_provider=datasource_provider,- )- logger.info(- "Refreshed datasource credentials for provider %s",- provider_name,- extra={- "tenant_id": tenant_id,- "plugin_id": datasource_provider_id.plugin_id,- "provider": provider_name,- "credential_id": credential_id,- "credential_name": credential_name,- "expires_at": refreshed_credentials.expires_at,- },- )- return encrypted_credentials, refreshed_credentials.expires_at-def remove_oauth_custom_client_params(self, tenant_id: str, datasource_provider_id: DatasourceProviderID):"""remove oauth custom client params@@ -259,36 +182,42 @@ class DatasourceProviderService:def _resolve_provider_credentials(self, record: DatasourceCredentialRecord, *, current_user: Any | None = None) -> dict[str, Any]:- """Refresh a snapshot; the store owns each read and conditional write transaction."""- if self._should_refresh_credentials(record):- encrypted, expires_at = self._refresh_datasource_credentials(- tenant_id=record.workspace_id,+ """Resolve through the same refresh policy and lease as actor and stored-document callers."""++ def load_latest() -> DatasourceCredentialRecord:+ latest = self._credentials.get_provider_credential(+ workspace_id=record.workspace_id,provider=record.provider,plugin_id=record.plugin_id,- datasource_provider=record,- current_user=current_user if current_user is not None else get_current_user(),+ credential_id=record.id,)- saved = self._credentials.update_if_unchanged(- record=record, encrypted_credentials=encrypted, expires_at=expires_at+ if latest is None:+ raise DatasourceCredentialNotFoundError()+ return latest++ actor_id = current_user.id if current_user is not None else None+ if (+ actor_id is None+ and record.owner_id is None+ and credential_needs_refresh(record.expires_at, now=time.time())+ ):+ actor_id = get_current_user().id+ try:+ return resolve_credential_snapshot(+ record,+ actor_id=actor_id,+ codec=self._codec,+ refresher=self._refresher,+ update=self._credentials.update_if_unchanged,+ load_latest=load_latest,+ refresh_lock=self._refresh_lock,+ now=time.time,)- if saved:- record = replace(record, encrypted_credentials=encrypted, expires_at=expires_at)- else:- latest = self._credentials.get_provider_credential(- workspace_id=record.workspace_id,- provider=record.provider,- plugin_id=record.plugin_id,- credential_id=record.id,- )- if latest is None or self._should_refresh_credentials(latest):- raise ValueError("Credential changed while it was being refreshed")- record = latest- return self.decrypt_datasource_provider_credentials(- tenant_id=record.workspace_id,- datasource_provider=record,- plugin_id=record.plugin_id,- provider=record.provider,- )+ except DatasourceCredentialError as error:+ raise ValueError(+ f"Failed to refresh datasource credentials for provider {record.provider}"+ f" (credential: {record.name or record.id}): {error}"+ ) from errordef get_all_datasource_credentials_by_provider(self,diff --git a/api/services/knowledge/indexing/adapters/estimate.py b/api/services/knowledge/indexing/adapters/estimate.pyindex d6279f2d2ea..08233aab83c 100644--- a/api/services/knowledge/indexing/adapters/estimate.py+++ b/api/services/knowledge/indexing/adapters/estimate.py@@ -26,6 +26,7 @@ from graphon.model_runtime.entities.model_entities import ModelTypefrom models.dataset import Dataset, DatasetProcessRulefrom repositories.knowledge.dataset_repository import _get_datasetfrom repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository+from services.credentials.refresh import CredentialRefreshUnavailableErrorfrom services.knowledge.indexing.estimate import (IndexingEstimateExecutionError,IndexingEstimateProviderUnavailableError,@@ -87,6 +88,8 @@ class IndexingEstimateAdapter:)except (LLMBadRequestError, ProviderTokenNotInitError, PluginDaemonClientSideError) as error:raise IndexingEstimateProviderUnavailableError(error.description) from error+ except CredentialRefreshUnavailableError:+ raiseexcept Exception as error:raise IndexingEstimateExecutionError(str(error)) from errordiff --git a/api/services/knowledge/indexing/estimate.py b/api/services/knowledge/indexing/estimate.pyindex 6e75705e27c..d9c1b5d98f4 100644--- a/api/services/knowledge/indexing/estimate.py+++ b/api/services/knowledge/indexing/estimate.py@@ -6,6 +6,7 @@ from typing import NamedTuple, Never, Protocolfrom core.entities.knowledge_entities import IndexingEstimatefrom core.rag.extractor.entity.extract_setting import ExtractSettingfrom machinery.context import RequestContext+from services.credentials.refresh import CredentialRefreshUnavailableErrorfrom services.data_source.credential_gateway import DatasourceCredentialErrorfrom services.data_source.entities.notion_import import NotionPageTypefrom services.knowledge.dataset_access import DatasetAccess, DatasetNotFoundError@@ -328,7 +329,7 @@ class IndexingEstimateApplicationService:@staticmethoddef _translate_error(error: Exception) -> Never:- if isinstance(error, IndexingEstimateApplicationError):+ if isinstance(error, (IndexingEstimateApplicationError, CredentialRefreshUnavailableError)):raise errorif isinstance(error, UnsupportedStoredSourceError):raise UnsupportedEstimateSourceError(error.source_type) from errordiff --git a/api/tests/helpers/unit/credential_refresh.py b/api/tests/helpers/unit/credential_refresh.pynew file mode 100644index 00000000000..ed98734157b--- /dev/null+++ b/api/tests/helpers/unit/credential_refresh.py@@ -0,0 +1,72 @@+"""Concrete credential ports for unit tests, including actual thread exclusion."""++from __future__ import annotations++from collections.abc import Generator+from contextlib import contextmanager+from threading import Condition, Lock+from typing import TYPE_CHECKING, Literal++from services.credentials.refresh import CredentialRefreshLease, CredentialRefreshUnavailableError+from services.data_source.credential_gateway import DatasourceProviderCredentialStore++if TYPE_CHECKING:+ from services.data_source.provider_service import DatasourceProviderService+++class ThreadCredentialRefreshLock:+ def __init__(self, *, wait_seconds: float = 2) -> None:+ self.wait_seconds = wait_seconds+ self.keys: list[tuple[str, str, str]] = []+ self.lost = False+ self.unavailable = False+ self._condition = Condition()+ self._locks: dict[tuple[str, str, str], Lock] = {}++ @contextmanager+ def acquire(+ self, *, credential_type: Literal["tool", "datasource"], workspace_id: str, credential_id: str+ ) -> Generator[CredentialRefreshLease]:+ key = (credential_type, workspace_id, credential_id)+ with self._condition:+ self.keys.append(key)+ lock = self._locks.setdefault(key, Lock())+ self._condition.notify_all()+ if self.unavailable or not lock.acquire(timeout=self.wait_seconds):+ raise CredentialRefreshUnavailableError()+ try:+ yield self+ finally:+ lock.release()++ def ensure_owned(self) -> None:+ if self.lost:+ raise CredentialRefreshUnavailableError()++ def wait_for_callers(self, count: int) -> None:+ """Keep the provider request open until competing callers have attempted the lock."""+ with self._condition:+ assert self._condition.wait_for(lambda: len(self.keys) >= count, timeout=self.wait_seconds)+++def build_datasource_provider_service(*, credentials: DatasourceProviderCredentialStore) -> DatasourceProviderService:+ from core.plugin.impl.datasource import PluginDatasourceManager+ from core.plugin.impl.oauth import OAuthHandler+ from services.data_source.credential_adapters import (+ OAuthDatasourceCredentialRefresher,+ PluginDatasourceCredentialCodec,+ PluginDatasourceOAuthClientResolver,+ )+ from services.data_source.provider_service import DatasourceProviderService++ manager = PluginDatasourceManager()+ return DatasourceProviderService(+ credentials=credentials,+ codec=PluginDatasourceCredentialCodec(provider_manager=manager),+ refresher=OAuthDatasourceCredentialRefresher(+ oauth_clients=PluginDatasourceOAuthClientResolver(configs=credentials, provider_manager=manager),+ oauth_handler=OAuthHandler(),+ ),+ refresh_lock=ThreadCredentialRefreshLock(),+ provider_manager=manager,+ )diff --git a/api/tests/unit_tests/controllers/common/test_errors.py b/api/tests/unit_tests/controllers/common/test_errors.pyindex 417882ceef0..90fb8f3e35d 100644--- a/api/tests/unit_tests/controllers/common/test_errors.py+++ b/api/tests/unit_tests/controllers/common/test_errors.py@@ -20,9 +20,11 @@ from controllers.common.errors import (UnsupportedFileTypeError,)from controllers.console import api as console_api+from controllers.inner_api import api as inner_apifrom controllers.service_api import api as service_apifrom libs.exception import BaseHTTPExceptionfrom libs.external_api import ExternalApi+from services.credentials.refresh import CredentialRefreshUnavailableErrorfrom services.errors.base import NoPermissionErrorfrom services.errors.workspace import WorkspaceApplicationError@@ -70,6 +72,25 @@ def test_permission_error_mapping_is_not_registered_on_generic_apis() -> None:assert response.json == {"code": "unknown", "message": "Internal Server Error", "status": 500}[email protected]("source_api", [console_api, inner_api], ids=["console", "inner_api"])+def test_refresh_unavailability_has_one_documented_response(source_api: ExternalApi) -> None:+ error = CredentialRefreshUnavailableError()+ app = Flask(__name__)+ api = Api(app, doc=False)+ api.error_handlers = source_api.error_handlers.copy()++ class Endpoint(Resource):+ def get(self) -> ResponseReturnValue:+ raise error++ api.add_resource(Endpoint, "/error")+ response = app.test_client().get("/error")++ assert response.status_code == 503+ assert response.json == {"code": "service_unavailable", "message": str(error), "status": 503}+ assert source_api.resolve_error(CredentialRefreshUnavailableError) == (503, "service_unavailable")++class TestFilenameNotExistsError:def test_defaults(self):error = FilenameNotExistsError()diff --git a/api/tests/unit_tests/controllers/console/datasets/test_data_source.py b/api/tests/unit_tests/controllers/console/datasets/test_data_source.pyindex e887b5f8fa7..2f62faf8b7d 100644--- a/api/tests/unit_tests/controllers/console/datasets/test_data_source.py+++ b/api/tests/unit_tests/controllers/console/datasets/test_data_source.py@@ -11,10 +11,10 @@ from typing import TYPE_CHECKING, castimport pytestfrom flask import Flaskfrom flask.testing import FlaskClient-from flask_restx import Apifrom controllers.common.errors import NotFoundErrorfrom controllers.common.rbac import DatasetId+from controllers.console import api as console_apifrom controllers.console import console_nsfrom controllers.console.app.error import ProviderNotInitializeErrorfrom controllers.console.datasets import data_source as data_source_controller@@ -22,8 +22,10 @@ from controllers.console.datasets.error import IndexingEstimateErrorfrom core.entities.knowledge_entities import IndexingEstimatefrom core.rbac import RBACPermissionfrom enums import DeploymentEdition+from libs.external_api import ExternalApifrom libs.login import AccountWithTenantfrom machinery.context import RequestContext+from services.credentials.refresh import CredentialRefreshUnavailableErrorfrom services.data_source.binding_application_service import (BindingMutationResult,DataSourceBindingApplicationService,@@ -234,6 +236,7 @@ class RecordingNotionSource:workspaces: tuple[NotionWorkspace, ...]list_calls: list[tuple[str, str, str]] = field(default_factory=list)preview_calls: list[tuple[str, str, str, str, str]] = field(default_factory=list)+ error: Exception | None = Nonedef list_authorized_pages(self,@@ -243,6 +246,8 @@ class RecordingNotionSource:credential_id: str,) -> tuple[NotionWorkspace, ...]:self.list_calls.append((workspace_id, actor_id, credential_id))+ if self.error is not None:+ raise self.errorreturn self.workspacesdef preview_page(@@ -255,6 +260,8 @@ class RecordingNotionSource:page_type: str,) -> str:self.preview_calls.append((workspace_id, actor_id, credential_id, page_id, page_type))+ if self.error is not None:+ raise self.errorreturn "preview text"@@ -328,7 +335,9 @@ class TestDataSourceCases:)app = Flask(__name__)app.config.update(TESTING=False, PROPAGATE_EXCEPTIONS=False)- Api(app).add_namespace(console_ns, path="/console/api")+ api = ExternalApi(app)+ api.error_handlers = console_api.error_handlers.copy()+ api.add_namespace(console_ns, path="/console/api")account = AdmittedAccount(id=ACCOUNT_ID, role="owner", status="active")account_with_tenant = AccountWithTenant(account=cast("Account", account), tenant_id=TENANT_ID)@@ -581,6 +590,21 @@ class TestDataSourceCases:(TENANT_ID, ACCOUNT_ID, "credential-1", PAGE_ID, page_type)]+ @staticmethod+ @pytest.mark.parametrize("path", ["/notion/pre-import/pages", f"/notion/pages/{PAGE_ID}/page/preview"])+ def test_notion_refresh_unavailable_returns_retryable_response(+ collaborators: ControllerCollaborators, path: str+ ) -> None:+ error = CredentialRefreshUnavailableError()+ collaborators.notion_source.error = error+ response = collaborators.client.get(f"/console/api{path}", query_string={"credential_id": "credential-1"})+ assert response.status_code == 503+ assert response.get_json() == {+ "code": "service_unavailable",+ "message": str(error),+ "status": 503,+ }+@staticmethoddef test_notion_estimate_maps_nested_payload_to_shared_command(collaborators: ControllerCollaborators) -> None:response = collaborators.client.post(diff --git a/api/tests/unit_tests/controllers/inner_api/agent/test_tools.py b/api/tests/unit_tests/controllers/inner_api/agent/test_tools.pyindex 3723ecf8a10..ce082dcfc0b 100644--- a/api/tests/unit_tests/controllers/inner_api/agent/test_tools.py+++ b/api/tests/unit_tests/controllers/inner_api/agent/test_tools.py@@ -2,11 +2,26 @@from collections.abc import Generatorfrom contextlib import contextmanager-from unittest.mock import patch+from unittest.mock import MagicMock, patch+import pytestfrom flask import Flask+from redis.exceptions import LockNotOwnedError, RedisError+from redis.lock import Lock+from sqlalchemy.orm import Sessionfrom controllers.inner_api import bp as inner_api_bp+from core.plugin.entities.plugin_daemon import CredentialType+from core.tools.entities.tool_entities import ToolProviderEntityWithPlugin+from core.tools.plugin_tool.provider import PluginToolProviderController+from core.tools.tool_manager import ToolManager+from extensions.application_services.tools import resolve_tool_credentials+from extensions.ext_redis import RedisClientWrapper+from models.model import App, AppMode+from models.tools import BuiltinToolProvider+from services.credentials.refresh import CredentialRefreshUnavailableError+from services.credentials.tool_runtime import ToolCredentialRefreshError+from services.credentials.tool_runtime_adapters import ProviderToolCredentialRuntimefrom services.entities.agent_tool_inner import AgentToolInvokeResponsefrom services.errors.agent_tool_inner import AgentToolInnerServiceErrorfrom tests.helpers.unit.config_override import config_overrides_context@@ -19,7 +34,7 @@ def _headers(api_key: str | None = "inner-key") -> dict[str, str]:return headers-def _payload() -> dict[str, object]:+def _payload(*, provider_type: str = "plugin", credential_id: str | None = "credential-1") -> dict[str, object]:return {"caller": {"tenant_id": "tenant-1",@@ -36,10 +51,10 @@ def _payload() -> dict[str, object]:"agent_config_version_id": "snapshot-1",},"tool": {- "provider_type": "plugin",+ "provider_type": provider_type,"provider_id": "langgenius/search/search","tool_name": "search",- "credential_id": "credential-1",+ "credential_id": credential_id,"tool_parameters": {"query": "dify"},"runtime_parameters": {"region": "us"},},@@ -122,3 +137,115 @@ def test_post_preserves_service_error_status_code_and_description() -> None:body = response.get_json()assert body["code"] == "app_tenant_mismatch"assert body["message"] == "App does not belong to the caller tenant."++[email protected]("credential_id", [None, "1a1db43c-e9d8-4ef2-af8f-38b25424a4ad"])[email protected]("failure", ["busy", "redis", "lost", "rejected"])+def test_post_preserves_refresh_error_classification_through_real_credential_resolution(+ sqlite_session: Session,+ redis_transport: tuple[RedisClientWrapper, MagicMock],+ monkeypatch: pytest.MonkeyPatch,+ credential_id: str | None,+ failure: str,+) -> None:+ provider_id = "langgenius/search/search"+ credential = BuiltinToolProvider(+ tenant_id="tenant-1",+ user_id="user-1",+ provider=provider_id,+ name="Search",+ credential_type=CredentialType.OAUTH2,+ encrypted_credentials="{}",+ expires_at=1,+ is_default=True,+ )+ credential.id = "1a1db43c-e9d8-4ef2-af8f-38b25424a4ad"+ sqlite_session.add_all(+ [+ App(+ id="app-1",+ tenant_id="tenant-1",+ name="Agent",+ mode=AppMode.AGENT_CHAT,+ enable_site=False,+ enable_api=False,+ max_active_requests=None,+ ),+ credential,+ ]+ )+ sqlite_session.commit()+ controller = PluginToolProviderController(+ entity=ToolProviderEntityWithPlugin.model_validate(+ {+ "identity": {+ "author": "langgenius",+ "name": "search",+ "description": {"en_US": "Search"},+ "icon": "search.svg",+ "label": {"en_US": "Search"},+ },+ "oauth_schema": {+ "client_schema": [],+ "credentials_schema": [+ {"name": "access_token", "type": "secret-input", "label": {"en_US": "Access token"}}+ ],+ },+ "tools": [+ {+ "identity": {+ "author": "langgenius",+ "name": "search",+ "provider": "search",+ "label": {"en_US": "Search"},+ },+ "description": {"human": {"en_US": "Search"}, "llm": "Search"},+ "parameters": [],+ }+ ],+ }+ ),+ plugin_id="langgenius/search",+ plugin_unique_identifier="langgenius/search:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ tenant_id="tenant-1",+ )+ redis, _ = redis_transport+ monkeypatch.setattr("extensions.application_services.tools.redis_client", redis)+ monkeypatch.setattr("core.helper.provider_cache.redis_client", redis)+ monkeypatch.setattr(ToolManager, "_credential_resolver", resolve_tool_credentials)+ app = Flask(__name__)+ app.config["TESTING"] = True+ app.register_blueprint(inner_api_bp)+ rejected = ToolCredentialRefreshError("OAuth credential was rejected. Reauthorize the provider.")++ with (+ _agent_inner_auth(),+ patch.object(ToolManager, "get_builtin_provider", return_value=controller),+ patch("core.helper.credential_utils.check_credential_policy_compliance"),+ patch.object(+ Lock,+ "acquire",+ return_value=failure != "busy",+ side_effect=RedisError("unavailable") if failure == "redis" else None,+ ),+ patch.object(Lock, "reacquire", side_effect=LockNotOwnedError("expired") if failure == "lost" else None),+ patch.object(Lock, "release"),+ patch.object(ProviderToolCredentialRuntime, "refresh", side_effect=rejected) as refresh,+ ):+ response = app.test_client().post(+ "/inner/api/agent/tools/invoke",+ json=_payload(provider_type="builtin", credential_id=credential_id),+ headers=_headers(),+ )++ expected_status = 422 if failure == "rejected" else 503+ assert response.status_code == expected_status+ assert response.get_json() == {+ "code": "agent_tool_credential_invalid" if failure == "rejected" else "service_unavailable",+ "message": str(rejected) if failure == "rejected" else str(CredentialRefreshUnavailableError()),+ "status": expected_status,+ }+ assert refresh.call_count == (1 if failure == "rejected" else 0)+ sqlite_session.refresh(credential)+ assert credential.encrypted_credentials == "{}"+ assert credential.expires_at == 1diff --git a/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py b/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.pyindex 0112481074c..4d78dd4e98a 100644--- a/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py+++ b/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py@@ -43,7 +43,6 @@ from models.enums import CreatorUserRolefrom models.model import UploadFilefrom repositories.credentials.query_repository import CredentialQueryRepositoryfrom repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository-from services.data_source.provider_service import DatasourceProviderServicefrom services.errors.file import FileTooLargeError as FileTooLargeServiceErrorfrom services.errors.file import UnsupportedFileTypeErrorfrom services.rag_pipeline.entity.pipeline_service_api_entities import (@@ -51,6 +50,7 @@ from services.rag_pipeline.entity.pipeline_service_api_entities import (PipelineRunApiEntity,)from services.rag_pipeline.rag_pipeline import RagPipelineService+from tests.helpers.unit.credential_refresh import build_datasource_provider_servicefrom tests.helpers.unit.model_factories import make_upload_file"""@@ -101,7 +101,7 @@ class TestRagPipelineWorkflowCases:session_proxy = scoped_session(sqlite_session_factory)monkeypatch.setattr(workflow_module.db, "session", session_proxy)query = CredentialQueryRepository(session_factory=sqlite_session_factory)- providers = DatasourceProviderService(+ providers = build_datasource_provider_service(credentials=SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory))monkeypatch.setattr(diff --git a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.pyindex ec3a12bf1bb..925eb1fb7fd 100644--- a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py+++ b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py@@ -21,8 +21,8 @@ from models.model import EndUserfrom models.workflow import Workflow, WorkflowTypefrom repositories.knowledge.document_repository import SQLAlchemyDocumentRepositoryfrom services.data_source.credential_gateway import DatasourceProviderCredentialStore-from services.data_source.provider_service import DatasourceProviderServicefrom services.workflow_run_agg import WorkflowRunAgg+from tests.helpers.unit.credential_refresh import build_datasource_provider_serviceTENANT_ID = "00000000-0000-0000-0000-000000000001"PIPELINE_ID = "00000000-0000-0000-0000-000000000002"@@ -47,7 +47,7 @@ def generator(mocker: MockerFixture, sqlite_engine: Engine):gen = module.PipelineGenerator(execution_driver=WorkflowRunAgg.run,documents=SQLAlchemyDocumentRepository(session_factory=sessionmaker(bind=sqlite_engine)),- datasource_providers=DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)diff --git a/api/tests/unit_tests/core/datasource/website_crawl/test_website_crawl_plugin.py b/api/tests/unit_tests/core/datasource/website_crawl/test_website_crawl_plugin.pyindex acf2f4a8446..24b80342fe8 100644--- a/api/tests/unit_tests/core/datasource/website_crawl/test_website_crawl_plugin.py+++ b/api/tests/unit_tests/core/datasource/website_crawl/test_website_crawl_plugin.py@@ -1,11 +1,10 @@from collections.abc import Generator-from unittest.mock import Mock, create_autospec, patch+from unittest.mock import Mock, patchimport httpximport pytestfrom pytest_mock import MockerFixture-import services.data_source.website_service as website_service_modulefrom core.datasource.__base.datasource_runtime import DatasourceRuntimefrom core.datasource.entities.datasource_entities import (DatasourceEntity,@@ -18,7 +17,6 @@ from core.datasource.entities.datasource_entities import (from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePluginfrom core.datasource.website_crawl.website_crawl_provider import WebsiteCrawlDatasourcePluginProviderControllerfrom core.rag.extractor.watercrawl.provider import WaterCrawlProvider-from services.data_source.credential_gateway import DatasourceProviderCredentialStorefrom services.data_source.website_service import CrawlOptions, CrawlRequest, WebsiteServicefrom tests.helpers.unit.core.datasource.factories import datasource_entity@@ -516,11 +514,7 @@ class TestWebsiteCrawlCases:options={"limit": 10, "crawl_sub_pages": True, "only_main_content": True},)- result = WebsiteService(- providers=website_service_module.DatasourceProviderService(- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)- )- ).crawl_url(api_request)+ result = WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)assert result["status"] == "active"assert result["job_id"] == "job-123"@@ -550,11 +544,7 @@ class TestWebsiteCrawlCases:options={"limit": 20, "crawl_sub_pages": True, "max_depth": 2},)- result = WebsiteService(- providers=website_service_module.DatasourceProviderService(- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)- )- ).crawl_url(api_request)+ result = WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)assert result["status"] == "active"assert result["job_id"] == "job-456"@@ -585,11 +575,7 @@ class TestWebsiteCrawlCases:options={"limit": 15, "crawl_sub_pages": True, "use_sitemap": True},)- result = WebsiteService(- providers=website_service_module.DatasourceProviderService(- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)- )- ).crawl_url(api_request)+ result = WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)assert result["status"] == "active"assert result["job_id"] == "task-789"@@ -880,11 +866,7 @@ class TestWebsiteCrawlCases:# The error should be raised when trying to crawl with invalid providerwith pytest.raises(ValueError, match="Invalid provider"):- WebsiteService(- providers=website_service_module.DatasourceProviderService(- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)- )- ).crawl_url(api_request)+ WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)def test_missing_api_key_error(self, mocker: MockerFixture) -> None:"""Test that missing API key is handled properly at the httpx client level."""@@ -1311,11 +1293,7 @@ class TestWebsiteCrawlCases:},)- result = WebsiteService(- providers=website_service_module.DatasourceProviderService(- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)- )- ).crawl_url(api_request)+ result = WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)assert result["status"] == "active"assert result["job_id"] == "prompt-job-123"@@ -1361,11 +1339,7 @@ class TestWebsiteCrawlCases:provider="jinareader", url="https://example.com/page", options={"crawl_sub_pages": False, "limit": 1})- result = WebsiteService(- providers=website_service_module.DatasourceProviderService(- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)- )- ).crawl_url(api_request)+ result = WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)# In single-page mode, JinaReader returns data immediatelyassert result["status"] == "active"@@ -1407,11 +1381,7 @@ class TestWebsiteCrawlCases:},)- result = WebsiteService(- providers=website_service_module.DatasourceProviderService(- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)- )- ).crawl_url(api_request)+ result = WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)assert result["status"] == "active"assert result["job_id"] == "tag-filter-job"diff --git a/api/tests/unit_tests/core/tools/test_tool_manager.py b/api/tests/unit_tests/core/tools/test_tool_manager.pyindex e94d01f521f..f675fab54e2 100644--- a/api/tests/unit_tests/core/tools/test_tool_manager.py+++ b/api/tests/unit_tests/core/tools/test_tool_manager.py@@ -246,6 +246,10 @@ def _setup_list_providers_from_api_mocks(@pytest.fixture(autouse=True)def _reset_tool_manager_state():+ from extensions.application_services.tools import resolve_tool_credentials++ old_credential_resolver = ToolManager._credential_resolver+ ToolManager.configure_credential_resolver(resolve_tool_credentials)old_hardcoded = ToolManager._hardcoded_providers.copy()old_loaded = ToolManager._builtin_providers_loadedold_labels = ToolManager._builtin_tools_labels.copy()@@ -254,6 +258,7 @@ def _reset_tool_manager_state():try:yieldfinally:+ ToolManager._credential_resolver = old_credential_resolverToolManager._hardcoded_providers = old_hardcodedToolManager._builtin_providers_loaded = old_loadedToolManager._builtin_tools_labels = old_labels@@ -405,7 +410,9 @@ def test_get_tool_runtime_builtin_with_credentials_decrypts_and_forks(encrypter = Mock()encrypter.decrypt.return_value = {"api_key": "secret"}cache = Mock()- with patch("core.tools.tool_manager.create_provider_encrypter", return_value=(encrypter, cache)):+ with patch(+ "services.credentials.tool_runtime_adapters.create_provider_encrypter", return_value=(encrypter, cache)+ ):result = ToolManager.get_tool_runtime(provider_type=ToolProviderType.BUILT_IN,provider_id="time",@@ -419,16 +426,16 @@ def test_get_tool_runtime_builtin_with_credentials_decrypts_and_forks(assert runtime.credential_type == CredentialType.API_KEY-@patch("core.tools.tool_manager.create_provider_encrypter")-@patch("core.plugin.impl.oauth.OAuthHandler")+@patch("services.credentials.tool_runtime_adapters.create_provider_encrypter")+@patch("services.credentials.tool_runtime_adapters.OAuthHandler")@patch("services.tools.builtin_tools_manage_service.BuiltinToolManageService.get_oauth_client",return_value=PluginOAuthClientConfig("langgenius/time:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"client_id": "id"}),)-@patch("core.tools.tool_manager.time.time", return_value=1000)+@patch("services.credentials.tool_runtime.time.time", return_value=1000)@patch("core.helper.credential_utils.check_credential_policy_compliance")@pytest.mark.parametrize("client_credentials", [None, {}, {"client_id": "id"}])[email protected]("plugin_provider", [False, True])[email protected]("credential_selection", ["builtin", "plugin-explicit", "plugin-default"])def test_get_tool_runtime_builtin_refreshes_expired_oauth_credentials(mock_check,mock_time,@@ -438,7 +445,7 @@ def test_get_tool_runtime_builtin_refreshes_expired_oauth_credentials(monkeypatch: pytest.MonkeyPatch,tool_database: _ToolDatabase,client_credentials: dict[str, str] | None,- plugin_provider: bool,+ credential_selection: str,):tool = Mock()tool.fork_tool_runtime.return_value = "runtime-tool"@@ -452,6 +459,7 @@ def test_get_tool_runtime_builtin_refreshes_expired_oauth_credentials(mock_get_oauth_client.return_value = (PluginOAuthClientConfig(identifier, client_credentials) if client_credentials is not None else None)+ plugin_provider = credential_selection != "builtin"if plugin_provider:controller = PluginToolProviderController(entity=ToolProviderEntityWithPlugin.model_validate(@@ -496,20 +504,18 @@ def test_get_tool_runtime_builtin_refreshes_expired_oauth_credentials(provider_id="time",tool_name="weekday",tenant_id=tenant_id,- credential_id=provider_id if plugin_provider else None,+ credential_id=provider_id if credential_selection == "plugin-explicit" else None,)assert result == "runtime-tool"- assert builtin_provider.expires_at == refreshed.expires_at- assert builtin_provider.encrypted_credentials == json.dumps({"token": "encrypted"})tool_database.session.expire_all()persisted = tool_database.session.get(BuiltinToolProvider, provider_id)assert persisted is not Noneassert persisted.expires_at == refreshed.expires_atassert persisted.encrypted_credentials == json.dumps({"token": "encrypted"})- cache.delete.assert_called_once()refresh = mock_oauth_handler_cls.return_value.refresh_credentialsrefresh.assert_called_once()+ assert refresh.call_args.kwargs["user_id"] == builtin_provider.user_idassert refresh.call_args.kwargs["system_credentials"] == (client_credentials or {})expected_identifier = identifier if plugin_provider or client_credentials is not None else Noneassert refresh.call_args.kwargs["plugin_unique_identifier"] == expected_identifier@@ -539,15 +545,15 @@ def test_get_tool_runtime_builtin_maps_oauth_refresh_failure_to_credential_errorencrypter.decrypt.return_value = {"token": "expired"}with (patch.object(ToolManager, "get_builtin_provider", return_value=controller),- patch("core.tools.tool_manager.create_provider_encrypter", return_value=(encrypter, Mock())),- patch("core.tools.tool_manager.time.time", return_value=1000),+ patch("services.credentials.tool_runtime_adapters.create_provider_encrypter", return_value=(encrypter, Mock())),+ patch("services.credentials.tool_runtime.time.time", return_value=1000),patch("services.tools.builtin_tools_manage_service.BuiltinToolManageService.get_oauth_client",return_value=PluginOAuthClientConfig("langgenius/time:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"client_id": "id"}),),- patch("core.plugin.impl.oauth.OAuthHandler") as oauth_handler_cls,+ patch("services.credentials.tool_runtime_adapters.OAuthHandler") as oauth_handler_cls,):oauth_handler_cls.return_value.refresh_credentials.side_effect = ValueError("refresh token revoked")with pytest.raises(ToolProviderCredentialValidationError, match="could not be refreshed"):diff --git a/api/tests/unit_tests/extensions/test_ext_application_services.py b/api/tests/unit_tests/extensions/test_ext_application_services.pyindex ab18ed81c9c..2f8aea35211 100644--- a/api/tests/unit_tests/extensions/test_ext_application_services.py+++ b/api/tests/unit_tests/extensions/test_ext_application_services.py@@ -255,6 +255,10 @@ def test_init_app_registers_services_for_the_current_app(ext_application_services.init_app(app)+ from extensions.application_services.tools import resolve_tool_credentials++ assert ext_application_services.ToolManager._credential_resolver is resolve_tool_credentials+with app.app_context():services = ext_application_services.application_services()assert services is app.extensions["application_services"]diff --git a/api/tests/unit_tests/repositories/credentials/test_query_repository.py b/api/tests/unit_tests/repositories/credentials/test_query_repository.pyindex 9bb0ffab543..2d1b2cf44e4 100644--- a/api/tests/unit_tests/repositories/credentials/test_query_repository.py+++ b/api/tests/unit_tests/repositories/credentials/test_query_repository.py@@ -3,11 +3,12 @@Tests visibility filtering and complete domain reads against real SQLite rows."""+from dataclasses import replacefrom datetime import datetimefrom uuid import uuid4import pytest-from sqlalchemy import select+from sqlalchemy import delete, select, updatefrom sqlalchemy.orm import Session, sessionmakerfrom core.plugin.entities.plugin_daemon import CredentialType as TriggerCredentialType@@ -449,3 +450,107 @@ class TestApplyVisibilityFilter:)assert sqlite_session.scalars(query).all() == []+++def runtime_repository(sqlite_session_factory: sessionmaker[Session]) -> CredentialQueryRepository:+ with sqlite_session_factory.begin() as session:+ for index, (identity, provider, default) in enumerate(+ [+ ("oldest", "provider", False),+ ("default", "org/plugin/provider", True),+ ("other", "other", True),+ ]+ ):+ row = BuiltinToolProvider(+ tenant_id="workspace",+ user_id="owner",+ provider=provider,+ name=identity,+ credential_type=TriggerCredentialType.OAUTH2,+ encrypted_credentials='{"token":"old"}',+ expires_at=1,+ is_default=default,+ )+ row.id = identity+ row.created_at = datetime(2026, 1, index + 1)+ row.updated_at = datetime(2026, 1, 1)+ session.add(row)+ return CredentialQueryRepository(session_factory=sqlite_session_factory)+++def test_lookup_is_scoped_and_prefers_default(runtime_repository: CredentialQueryRepository) -> None:+ providers = ("org/plugin/provider", "provider")+ record = runtime_repository.get_tool_credential(workspace_id="workspace", providers=providers, credential_id=None)+ assert record is not None+ assert record.id == "default"+ assert (+ runtime_repository.get_tool_credential(workspace_id="other", providers=providers, credential_id="default")+ is None+ )+ assert (+ runtime_repository.get_tool_credential(workspace_id="workspace", providers=providers, credential_id="other")+ is None+ )+ oldest = runtime_repository.get_tool_credential(+ workspace_id="workspace", providers=("provider",), credential_id=None+ )+ assert oldest is not None+ assert oldest.id == "oldest"+++def test_refresh_commits_for_a_detached_snapshot(+ runtime_repository: CredentialQueryRepository, sqlite_session_factory: sessionmaker[Session]+) -> None:+ record = runtime_repository.get_tool_credential(+ workspace_id="workspace", providers=("provider",), credential_id="oldest"+ )+ assert record is not None+ assert runtime_repository.update_tool_credential_if_unchanged(+ record=record, encrypted_credentials='{"token":"new"}', expires_at=500+ )+ with sqlite_session_factory() as session:+ persisted = session.get(BuiltinToolProvider, record.id)+ assert persisted is not None+ assert persisted.encrypted_credentials == '{"token":"new"}'+ assert persisted.expires_at == 500+ assert persisted.updated_at > datetime(2026, 1, 1)+ assert record.encrypted_credentials == '{"token":"old"}'+ assert not runtime_repository.update_tool_credential_if_unchanged(+ record=record, encrypted_credentials="stale", expires_at=600+ )+++ ("field", "value"),+ [+ ("tenant_id", "other"),+ ("provider", "other"),+ ("user_id", "other"),+ ("credential_type", TriggerCredentialType.API_KEY),+ ("encrypted_credentials", "reauthorized"),+ ("expires_at", 2),+ ("deleted", None),+ ],+)+def test_refresh_preserves_concurrent_changes(+ runtime_repository: CredentialQueryRepository,+ sqlite_session_factory: sessionmaker[Session],+ field: str,+ value: object,+) -> None:+ record = runtime_repository.get_tool_credential(+ workspace_id="workspace", providers=("provider",), credential_id="oldest"+ )+ assert record is not None+ with sqlite_session_factory.begin() as session:+ statement = (+ delete(BuiltinToolProvider) if field == "deleted" else update(BuiltinToolProvider).values({field: value})+ )+ session.execute(statement.where(BuiltinToolProvider.id == record.id))+ assert not runtime_repository.update_tool_credential_if_unchanged(+ record=record, encrypted_credentials="refresh", expires_at=500+ )+ assert not runtime_repository.update_tool_credential_if_unchanged(+ record=replace(record, workspace_id="foreign"), encrypted_credentials="refresh", expires_at=500+ )diff --git a/api/tests/unit_tests/repositories/data_source/test_credential_repository.py b/api/tests/unit_tests/repositories/data_source/test_credential_repository.pyindex e330315dbe2..f83ea6165cf 100644--- a/api/tests/unit_tests/repositories/data_source/test_credential_repository.py+++ b/api/tests/unit_tests/repositories/data_source/test_credential_repository.py@@ -310,8 +310,14 @@ def test_get_oauth_client_config_returns_detached_tenant_and_system_rows(assert record.system_credentials == {"client_secret": "system-secret"}+ ("field", "value"),+ [("encrypted_credentials", {"integration_secret": "concurrent"}), ("expires_at", 200), ("auth_type", "api-key")],+)def test_update_if_unchanged_detects_concurrent_change(sqlite_session_factory: sessionmaker[Session],+ field: str,+ value: object,) -> None:repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)with sqlite_session_factory.begin() as session:@@ -332,8 +338,7 @@ def test_update_if_unchanged_detects_concurrent_change(with sqlite_session_factory.begin() as session:stored = session.scalar(select(DatasourceProvider).where(DatasourceProvider.id == credential_id))assert stored is not None- stored.encrypted_credentials = {"integration_secret": "concurrent"}- stored.expires_at = 200+ setattr(stored, field, value)assert not repository.update_if_unchanged(record=record,diff --git a/api/tests/unit_tests/services/credentials/test_redis_refresh_lock.py b/api/tests/unit_tests/services/credentials/test_redis_refresh_lock.pynew file mode 100644index 00000000000..febeec491dc--- /dev/null+++ b/api/tests/unit_tests/services/credentials/test_redis_refresh_lock.py@@ -0,0 +1,89 @@+"""Lease lifetime and failure behavior with the real redis-py Lock class."""++from threading import Event, Thread, current_thread+from unittest.mock import MagicMock, patch++import pytest+from redis.exceptions import LockNotOwnedError, RedisError+from redis.lock import Lock++from extensions.ext_redis import RedisClientWrapper+from services.credentials import redis_refresh_lock+from services.credentials.redis_refresh_lock import RedisCredentialRefreshLock+from services.credentials.refresh import CredentialRefreshUnavailableError+++def test_lease_renews_during_blocking_refresh_and_releases_on_error(+ redis_transport: tuple[RedisClientWrapper, MagicMock], monkeypatch: pytest.MonkeyPatch+) -> None:+ redis, _ = redis_transport+ renewed = Event()+ monkeypatch.setattr(redis_refresh_lock, "_RENEW_SECONDS", 0.001)+ with (+ patch.object(Lock, "acquire", autospec=True, return_value=True) as acquire,+ patch.object(Lock, "reacquire", side_effect=renewed.set),+ patch.object(Lock, "release") as release,+ ):++ def refresh() -> None:+ with RedisCredentialRefreshLock(client=redis).acquire(+ credential_type="tool", workspace_id="workspace", credential_id="credential"+ ) as lease:+ assert renewed.wait(timeout=1)+ lease.ensure_owned()+ release.assert_not_called()+ raise ValueError("provider failed")++ with pytest.raises(ValueError, match="provider failed"):+ refresh()+ release.assert_called_once()+ lock = acquire.call_args.args[0]+ assert lock.name == "oauth:credential-refresh:tool:workspace:credential"+ assert lock.timeout == 60+ assert lock.blocking_timeout == 30+ assert lock.thread_local is False++[email protected]("error", [LockNotOwnedError("expired"), RedisError("unavailable")])+def test_lost_heartbeat_fails_closed(+ redis_transport: tuple[RedisClientWrapper, MagicMock], monkeypatch: pytest.MonkeyPatch, error: RedisError+) -> None:+ redis, _ = redis_transport+ attempted = Event()+ threads: list[Thread] = []++ def fail_renewal() -> None:+ threads.append(current_thread())+ attempted.set()+ raise error++ monkeypatch.setattr(redis_refresh_lock, "_RENEW_SECONDS", 0.001)+ with (+ patch.object(Lock, "acquire", return_value=True),+ patch.object(Lock, "reacquire", side_effect=fail_renewal),+ patch.object(Lock, "release", side_effect=LockNotOwnedError("expired")),+ ):+ with RedisCredentialRefreshLock(client=redis).acquire(+ credential_type="datasource", workspace_id="workspace", credential_id="credential"+ ) as lease:+ assert attempted.wait(timeout=1)+ threads[0].join(timeout=1)+ with pytest.raises(CredentialRefreshUnavailableError):+ lease.ensure_owned()++[email protected]("error", [None, RedisError("unavailable")])+def test_busy_or_unavailable_lock_never_enters_refresh(+ redis_transport: tuple[RedisClientWrapper, MagicMock], error: RedisError | None+) -> None:+ redis, _ = redis_transport+ with (+ patch.object(Lock, "acquire", return_value=False, side_effect=error),+ patch.object(Lock, "release") as release,+ ):+ with pytest.raises(CredentialRefreshUnavailableError):+ with RedisCredentialRefreshLock(client=redis).acquire(+ credential_type="tool", workspace_id="workspace", credential_id="credential"+ ):+ pytest.fail("Refresh must not run without its lease")+ release.assert_not_called()diff --git a/api/tests/unit_tests/services/credentials/test_tool_runtime.py b/api/tests/unit_tests/services/credentials/test_tool_runtime.pynew file mode 100644index 00000000000..d5e45dd97e5--- /dev/null+++ b/api/tests/unit_tests/services/credentials/test_tool_runtime.py@@ -0,0 +1,162 @@+"""Refresh coordination with concrete ports and concurrent callers."""++import json+import time+from collections.abc import Callable, Mapping+from concurrent.futures import ThreadPoolExecutor+from dataclasses import dataclass, field, replace+from threading import Barrier++import pytest++from services.credentials.refresh import CredentialRefreshUnavailableError+from services.credentials.tool_runtime import (+ RefreshedToolCredential,+ ToolCredentialNotFoundError,+ ToolCredentialRefreshError,+ ToolCredentialService,+ ToolRuntimeCredential,+)+from tests.helpers.unit.credential_refresh import ThreadCredentialRefreshLock+++def _record(*, expires_at: int = 1) -> ToolRuntimeCredential:+ return ToolRuntimeCredential(+ "credential", "workspace", "owner", "provider", "oauth2", '{"token":"old"}', expires_at+ )+++@dataclass+class CredentialStore:+ record: ToolRuntimeCredential | None+ barrier: Barrier | None = None+ writes: int = 0++ def get_tool_credential(+ self, *, workspace_id: str, providers: tuple[str, ...], credential_id: str | None+ ) -> ToolRuntimeCredential | None:+ record = self.record+ if record is not None:+ assert record.workspace_id == workspace_id+ assert record.provider in providers+ assert credential_id in (None, record.id)+ if self.barrier is not None and credential_id is None:+ self.barrier.wait(timeout=2)+ return record++ def update_tool_credential_if_unchanged(+ self, *, record: ToolRuntimeCredential, encrypted_credentials: str, expires_at: int+ ) -> bool:+ if self.record != record:+ return False+ self.writes += 1+ self.record = replace(record, encrypted_credentials=encrypted_credentials, expires_at=expires_at)+ return True+++@dataclass+class CredentialRuntime:+ during_refresh: Callable[[], None] = lambda: None+ refreshes: int = 0+ invalidations: list[str] = field(default_factory=list)++ def check_policy(self, record: ToolRuntimeCredential) -> None:+ assert record.workspace_id == "workspace"++ def decrypt(self, record: ToolRuntimeCredential) -> dict[str, object]:+ return json.loads(record.encrypted_credentials or "{}")++ def encrypt(self, record: ToolRuntimeCredential, credentials: Mapping[str, object]) -> str:+ assert record.workspace_id == "workspace"+ return json.dumps(credentials)++ def refresh(self, record: ToolRuntimeCredential, credentials: Mapping[str, object]) -> RefreshedToolCredential:+ assert record.workspace_id == "workspace"+ assert credentials == {"token": "old"}+ self.refreshes += 1+ self.during_refresh()+ return RefreshedToolCredential({"token": "new"}, int(time.time()) + 3600)++ def invalidate(self, record: ToolRuntimeCredential) -> None:+ self.invalidations.append(record.id)+++def _service(+ store: CredentialStore, runtime: CredentialRuntime, lock: ThreadCredentialRefreshLock+) -> ToolCredentialService:+ return ToolCredentialService(credentials=store, runtime=runtime, refresh_lock=lock, providers=("provider",))+++def test_concurrent_and_sequential_calls_refresh_once() -> None:+ store = CredentialStore(_record(), barrier=Barrier(4))+ lock = ThreadCredentialRefreshLock()+ runtime = CredentialRuntime(during_refresh=lambda: lock.wait_for_callers(4))+ service = _service(store, runtime, lock)+ with ThreadPoolExecutor(max_workers=4) as pool:+ futures = [pool.submit(service.resolve, workspace_id="workspace", credential_id=None) for _ in range(4)]+ assert [future.result(timeout=3).credentials for future in futures] == [{"token": "new"}] * 4+ assert service.resolve(workspace_id="workspace", credential_id="credential").credentials == {"token": "new"}+ assert runtime.refreshes == store.writes == 1+ assert runtime.invalidations == ["credential"]+ assert lock.keys == [("tool", "workspace", "credential")] * 4++[email protected]("expires_at", [-1, int(time.time()) + 3600])+def test_valid_credentials_do_not_acquire_a_lock(expires_at: int) -> None:+ store, runtime, lock = (+ CredentialStore(_record(expires_at=expires_at)),+ CredentialRuntime(),+ ThreadCredentialRefreshLock(),+ )+ assert _service(store, runtime, lock).resolve(workspace_id="workspace", credential_id=None).credentials == {+ "token": "old"+ }+ assert lock.keys == []+ assert runtime.refreshes == store.writes == 0++[email protected]("failure", ["unavailable", "lost-before", "lost-after", "provider"])+def test_refresh_failure_does_not_write_or_return_expired_credentials(failure: str) -> None:+ store, runtime, lock = CredentialStore(_record()), CredentialRuntime(), ThreadCredentialRefreshLock()+ lock.unavailable = failure == "unavailable"+ lock.lost = failure == "lost-before"++ def fail_refresh() -> None:+ if failure == "lost-after":+ lock.lost = True+ if failure == "provider":+ raise ToolCredentialRefreshError("provider failed")++ runtime.during_refresh = fail_refresh+ with pytest.raises((ToolCredentialRefreshError, CredentialRefreshUnavailableError)):+ _service(store, runtime, lock).resolve(workspace_id="workspace", credential_id=None)+ assert store.writes == 0+ assert runtime.invalidations == []+ assert runtime.refreshes == (1 if failure in {"lost-after", "provider"} else 0)++[email protected]("change", ["reauthorized", "expired", "deleted"])+def test_refresh_cannot_overwrite_concurrent_changes(change: str) -> None:+ record = _record()+ store, runtime, lock = CredentialStore(record), CredentialRuntime(), ThreadCredentialRefreshLock()++ def change_credential() -> None:+ store.record = (+ None+ if change == "deleted"+ else replace(+ record,+ encrypted_credentials='{"token":"replacement"}',+ expires_at=-1 if change == "reauthorized" else 1,+ )+ )++ runtime.during_refresh = change_credential+ service = _service(store, runtime, lock)+ if change == "reauthorized":+ assert service.resolve(workspace_id="workspace", credential_id=None).credentials == {"token": "replacement"}+ else:+ with pytest.raises(ToolCredentialNotFoundError if change == "deleted" else ToolCredentialRefreshError):+ service.resolve(workspace_id="workspace", credential_id=None)+ assert store.writes == 0+ assert runtime.invalidations == []diff --git a/api/tests/unit_tests/services/credentials/test_tool_runtime_adapters.py b/api/tests/unit_tests/services/credentials/test_tool_runtime_adapters.pynew file mode 100644index 00000000000..e48ef5ee396--- /dev/null+++ b/api/tests/unit_tests/services/credentials/test_tool_runtime_adapters.py@@ -0,0 +1,115 @@+"""OAuth cache entries are bounded and checked against the persisted snapshot."""++import json+from dataclasses import replace++import pytest++from core.helper.provider_cache import ToolProviderCredentialsCache+from core.tools.entities.tool_entities import ToolProviderEntityWithPlugin+from core.tools.plugin_tool.provider import PluginToolProviderController+from services.credentials.tool_runtime import ToolRuntimeCredential+from services.credentials.tool_runtime_adapters import ProviderToolCredentialRuntime+++class CredentialCacheRedis:+ def __init__(self) -> None:+ self.values: dict[str, bytes] = {}+ self.ttls: dict[str, int] = {}++ def get(self, key: str) -> bytes | None:+ return self.values.get(key)++ def setex(self, key: str, ttl: int, value: str) -> None:+ self.values[key] = value.encode()+ self.ttls[key] = ttl++ def delete(self, key: str) -> None:+ self.values.pop(key, None)+ self.ttls.pop(key, None)+++def _controller() -> PluginToolProviderController:+ schemas = [+ {"name": name, "type": "secret-input", "label": {"en_US": name}} for name in ("access_token", "refresh_token")+ ]+ return PluginToolProviderController(+ entity=ToolProviderEntityWithPlugin.model_validate(+ {+ "identity": {+ "author": "org",+ "name": "provider",+ "description": {"en_US": "Provider"},+ "icon": "icon.svg",+ "label": {"en_US": "Provider"},+ },+ "credentials_schema": schemas,+ "oauth_schema": {"credentials_schema": schemas, "client_schema": []},+ }+ ),+ plugin_id="org/plugin",+ plugin_unique_identifier="org/plugin:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",+ tenant_id="workspace",+ )++[email protected]("invalidate_from", ["runtime", "management"])+def test_oauth_cache_reuses_only_matching_snapshots_and_invalidates_both_entries(+ monkeypatch: pytest.MonkeyPatch, invalidate_from: str+) -> None:+ redis = CredentialCacheRedis()+ monkeypatch.setattr("core.helper.provider_cache.redis_client", redis)+ decryptions: list[str] = []++ def decrypt(workspace_id: str, value: str) -> str:+ assert workspace_id == "workspace"+ decryptions.append(value)+ return f"plain:{value}"++ monkeypatch.setattr("core.helper.encrypter.decrypt_token", decrypt)+ runtime = ProviderToolCredentialRuntime(+ controller=_controller(), provider_id="org/plugin/provider", console_api_url=""+ )+ old_credentials = {"access_token": "access-old", "refresh_token": "refresh-old"}+ old = ToolRuntimeCredential(+ "credential", "workspace", "owner", "org/plugin/provider", "oauth2", json.dumps(old_credentials), -1+ )+ legacy = ToolProviderCredentialsCache(tenant_id=old.workspace_id, provider=old.provider, credential_id=old.id)+ legacy.set({"token": "stale-legacy-token"})+ old_plaintext = {name: f"plain:{value}" for name, value in old_credentials.items()}+ for _ in range(10):+ assert runtime.decrypt(old) == old_plaintext+ assert len(decryptions) == 2++ new_credentials = {"access_token": "access-new", "refresh_token": "refresh-new"}+ latest = replace(old, encrypted_credentials=json.dumps(new_credentials))+ new_plaintext = {name: f"plain:{value}" for name, value in new_credentials.items()}+ assert runtime.decrypt(latest) == new_plaintext+ assert len(decryptions) == 4++ # A reader of the old snapshot may finish its cache write after a refresh.+ runtime._cache(old).set(old_plaintext)+ assert runtime.decrypt(latest) == new_plaintext+ assert len(decryptions) == 6+ assert runtime.decrypt(latest) == new_plaintext+ assert len(decryptions) == 6+ assert len(redis.values) == 2+ assert set(redis.ttls.values()) == {86400}+ assert runtime.decrypt(replace(latest, credential_type="api-key")) == {"token": "stale-legacy-token"}++ if invalidate_from == "runtime":+ runtime.invalidate(latest)+ else:+ legacy.delete()+ assert redis.values == {}+ assert redis.ttls == {}+++def test_oauth_cache_ignores_a_malformed_snapshot(monkeypatch: pytest.MonkeyPatch) -> None:+ redis = CredentialCacheRedis()+ monkeypatch.setattr("core.helper.provider_cache.redis_client", redis)+ cache = ToolProviderCredentialsCache(+ tenant_id="workspace", provider="provider", credential_id="credential", fingerprint="current"+ )+ redis.setex(cache.cache_key, 86400, '{"fingerprint":"current","credentials":[]}')+ assert cache.get() is Nonediff --git a/api/tests/unit_tests/services/data_source/test_credential_gateway.py b/api/tests/unit_tests/services/data_source/test_credential_gateway.pyindex a14669f1726..c7f1f37f7fc 100644--- a/api/tests/unit_tests/services/data_source/test_credential_gateway.py+++ b/api/tests/unit_tests/services/data_source/test_credential_gateway.py@@ -1,7 +1,10 @@+import timefrom collections.abc import Mapping+from concurrent.futures import ThreadPoolExecutorfrom dataclasses import dataclass, field, replacefrom datetime import datetime, timedelta-from typing import cast+from threading import Barrier, local+from typing import Never, cast, overrideimport pytestfrom sqlalchemy.orm import Session, sessionmaker@@ -20,6 +23,7 @@ from core.plugin.impl.oauth import OAuthHandlerfrom models.oauth import DatasourceOauthParamConfig, DatasourceOauthTenantParamConfigfrom models.provider_ids import DatasourceProviderIDfrom repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository+from services.credentials.refresh import CredentialRefreshUnavailableErrorfrom services.data_source.credential_adapters import (OAuthDatasourceCredentialRefresher,PluginDatasourceCredentialCodec,@@ -35,7 +39,9 @@ from services.data_source.credential_gateway import (TrustedStoredDatasourceCredentialGateway,)from services.data_source.entities.credential import DatasourceCredentialRecord+from services.data_source.provider_service import DatasourceProviderServicefrom services.plugin.dispatch_identity import PluginOAuthClientConfig+from tests.helpers.unit.credential_refresh import ThreadCredentialRefreshLockdef _record(@@ -77,7 +83,7 @@ class ScriptedCredentialStore:) -> DatasourceCredentialRecord | None:self.events.append("read")self.get_calls.append((workspace_id, actor_id, credential_id, provider, plugin_id))- return self.records.pop(0)+ return self.records.pop(0) if len(self.records) > 1 else self.records[0]def update_if_unchanged(self,@@ -280,6 +286,7 @@ def _gateway(credentials=store,codec=ReversibleCredentialCodec(),refresher=refresh,+ refresh_lock=ThreadCredentialRefreshLock(),now=lambda: 100,),refresh,@@ -309,6 +316,7 @@ def test_trusted_stored_source_resolver_uses_distinct_owner_chain_port() -> Nonecredentials=catalog,codec=ReversibleCredentialCodec(),refresher=RecordingCredentialRefresher(),+ refresh_lock=ThreadCredentialRefreshLock(),now=lambda: 100,)@@ -339,6 +347,7 @@ def test_trusted_stored_source_resolver_fails_closed_for_missing_credential() ->credentials=StoredCredentialCatalog(None),codec=ReversibleCredentialCodec(),refresher=RecordingCredentialRefresher(),+ refresh_lock=ThreadCredentialRefreshLock(),now=lambda: 100,)@@ -361,6 +370,7 @@ def test_trusted_stored_source_refreshes_with_persisted_credential_owner() -> Nocredentials=catalog,codec=ReversibleCredentialCodec(),refresher=refresher,+ refresh_lock=ThreadCredentialRefreshLock(),now=lambda: 100,)@@ -396,7 +406,7 @@ def test_refresh_runs_between_read_and_conditional_update() -> None:assert _resolve(gateway) == {"integration_secret": "refreshed"}assert refresher.calls == [("workspace-1", "actor-1", record, {"integration_secret": "plain:encrypted"})]assert store.updates == [(record, {"integration_secret": "cipher:refreshed"}, 500)]- assert events == ["read", "refresh", "update"]+ assert events == ["read", "read", "refresh", "update"]def test_concurrent_refresh_uses_newer_non_expiring_snapshot() -> None:@@ -426,6 +436,175 @@ def test_concurrent_refresh_fails_closed_if_latest_snapshot_is_still_expired() -_resolve(gateway)+def test_actor_stored_and_provider_callers_share_one_refresh() -> None:+ class ReaderState(local):+ first = True++ class SharedCatalog:+ record = _record(expires_at=1)+ barrier = Barrier(3)+ reader = ReaderState()+ writes = 0++ def _read(+ self, workspace_id: str, credential_id: str | None, provider: str, plugin_id: str+ ) -> DatasourceCredentialRecord:+ record = self.record+ assert (workspace_id, credential_id, provider, plugin_id) == (+ record.workspace_id,+ record.id,+ record.provider,+ record.plugin_id,+ )+ if self.reader.first:+ self.reader.first = False+ self.barrier.wait(timeout=2)+ return record++ def get_visible(+ self, *, workspace_id: str, actor_id: str, credential_id: str, provider: str, plugin_id: str+ ) -> DatasourceCredentialRecord:+ assert actor_id == "actor-1"+ return self._read(workspace_id, credential_id, provider, plugin_id)++ def get_for_stored_document(+ self,+ *,+ workspace_id: str,+ dataset_id: str,+ document_id: str,+ credential_id: str | None,+ provider: str,+ plugin_id: str,+ ) -> DatasourceCredentialRecord:+ assert (dataset_id, document_id) == ("dataset-1", "document-1")+ return self._read(workspace_id, credential_id, provider, plugin_id)++ def get_provider_credential(+ self, *, workspace_id: str, credential_id: str | None, provider: str, plugin_id: str+ ) -> DatasourceCredentialRecord:+ return self._read(workspace_id, credential_id, provider, plugin_id)++ def list_provider_credentials(+ self, *, workspace_id: str, provider: str, plugin_id: str+ ) -> list[DatasourceCredentialRecord]:+ return [self._read(workspace_id, self.record.id, provider, plugin_id)]++ def get_oauth_client_config(self, *, workspace_id: str, provider: str, plugin_id: str) -> Never:+ raise AssertionError(f"Unexpected OAuth lookup for {workspace_id}:{plugin_id}/{provider}")++ def update_if_unchanged(+ self, *, record: DatasourceCredentialRecord, encrypted_credentials: Mapping[str, object], expires_at: int+ ) -> bool:+ assert self.record == record+ self.writes += 1+ self.record = replace(record, encrypted_credentials=encrypted_credentials, expires_at=expires_at)+ return True++ class RotatingRefresher:+ calls = 0++ def refresh(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ record: DatasourceCredentialRecord,+ credentials: Mapping[str, object],+ ) -> RefreshedDatasourceCredential:+ assert (workspace_id, record.id) == ("workspace-1", "credential-1")+ assert actor_id in {"actor-1", "credential-owner-1"}+ assert credentials == {"integration_secret": "plain:encrypted"}+ self.calls += 1+ lock.wait_for_callers(3)+ return RefreshedDatasourceCredential({"integration_secret": "refreshed"}, int(time.time()) + 3600)++ catalog, codec, refresher, lock = (+ SharedCatalog(),+ ReversibleCredentialCodec(),+ RotatingRefresher(),+ ThreadCredentialRefreshLock(),+ )+ actor = ActorAwareDatasourceCredentialGateway(+ credentials=catalog, codec=codec, refresher=refresher, refresh_lock=lock+ )+ stored = TrustedStoredDatasourceCredentialGateway(+ credentials=catalog, codec=codec, refresher=refresher, refresh_lock=lock+ )+ provider = DatasourceProviderService(+ credentials=catalog,+ codec=codec,+ refresher=refresher,+ refresh_lock=lock,+ provider_manager=PluginDatasourceManager(),+ )+ with ThreadPoolExecutor(max_workers=3) as pool:+ futures = [+ pool.submit(_resolve, actor),+ pool.submit(+ stored.resolve_for_document,+ workspace_id="workspace-1",+ dataset_id="dataset-1",+ document_id="document-1",+ credential_id="credential-1",+ provider="notion_datasource",+ plugin_id="langgenius/notion_datasource",+ ),+ pool.submit(+ provider.get_datasource_credentials,+ "workspace-1",+ "notion_datasource",+ "langgenius/notion_datasource",+ "credential-1",+ ),+ ]+ results = [future.result(timeout=3) for future in futures]+ assert all(result["integration_secret"] in {"refreshed", "plain:cipher:refreshed"} for result in results)+ assert refresher.calls == catalog.writes == 1+ assert lock.keys == [("datasource", "workspace-1", "credential-1")] * 3++[email protected]("failure", ["unavailable", "lost-before", "lost-after"])+def test_refresh_lease_failure_never_writes_or_returns_expired_data(failure: str) -> None:+ lock = ThreadCredentialRefreshLock()+ lock.unavailable = failure == "unavailable"+ lock.lost = failure == "lost-before"++ class LosingRefresher(RecordingCredentialRefresher):+ @override+ def refresh(+ self,+ *,+ workspace_id: str,+ actor_id: str,+ record: DatasourceCredentialRecord,+ credentials: Mapping[str, object],+ ) -> RefreshedDatasourceCredential:+ result = super().refresh(+ workspace_id=workspace_id, actor_id=actor_id, record=record, credentials=credentials+ )+ lock.lost = True+ return result++ store, refresher = ScriptedCredentialStore([_record(expires_at=1)]), LosingRefresher()+ gateway = ActorAwareDatasourceCredentialGateway(+ credentials=store, codec=ReversibleCredentialCodec(), refresher=refresher, refresh_lock=lock, now=lambda: 100+ )+ with pytest.raises(CredentialRefreshUnavailableError):+ _resolve(gateway)+ assert store.updates == []+ assert len(refresher.calls) == (1 if failure == "lost-after" else 0)+++def test_visibility_is_rechecked_after_waiting_for_the_refresh_lock() -> None:+ store = ScriptedCredentialStore([_record(expires_at=1), None])+ gateway, refresher = _gateway(store)+ with pytest.raises(DatasourceCredentialNotFoundError):+ _resolve(gateway)+ assert refresher.calls == []+ assert store.updates == []++@pytest.mark.parametrize(("auth_type", "secret_name"),[("api-key", "api_secret"), ("oauth2", "integration_secret")],diff --git a/api/tests/unit_tests/services/data_source/test_provider_service.py b/api/tests/unit_tests/services/data_source/test_provider_service.pyindex fe6dc2a054f..38618d4e1ec 100644--- a/api/tests/unit_tests/services/data_source/test_provider_service.py+++ b/api/tests/unit_tests/services/data_source/test_provider_service.py@@ -17,8 +17,10 @@ from models.provider_ids import DatasourceProviderIDfrom repositories.credentials.query_repository import CredentialQueryRepositoryfrom repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepositoryfrom services.data_source import provider_service as service_module+from services.data_source.credential_gateway import DatasourceCredentialRefreshError, RefreshedDatasourceCredentialfrom services.data_source.provider_service import DatasourceProviderService, get_current_userfrom services.plugin.dispatch_identity import PluginOAuthClientConfig+from tests.helpers.unit.credential_refresh import build_datasource_provider_service_PLUGIN_IDENTIFIER = f"org/plug:1.0.0@{'a' * 32}"@@ -105,7 +107,7 @@ class TestDatasourceProviderService:@pytest.fixturedef service(self, sqlite_session: Session):- return DatasourceProviderService(+ return build_datasource_provider_service(credentials=SQLAlchemyDatasourceCredentialRepository(session_factory=sessionmaker(bind=sqlite_session.get_bind(), expire_on_commit=False))@@ -132,9 +134,10 @@ class TestDatasourceProviderService:patch("core.plugin.impl.base._httpx_client.stream", side_effect=lambda **kw: httpx.stream(**kw)),patch("httpx.request") as mock_httpx,patch("services.data_source.provider_service.encrypter") as mock_enc,+ patch("services.data_source.credential_adapters.encrypter", new=mock_enc),patch("services.data_source.provider_service.redis_client") as mock_redis,patch("services.data_source.provider_service.generate_incremental_name") as mock_genname,- patch("services.data_source.provider_service.OAuthHandler") as mock_oauth,+ patch("core.plugin.impl.oauth.OAuthHandler") as mock_oauth,):mock_enc.encrypt_token.return_value = "enc_tok"mock_enc.decrypt_token.return_value = "dec_tok"@@ -348,7 +351,6 @@ class TestDatasourceProviderService:@pytest.mark.parametrize("oauth_client",[- None,PluginOAuthClientConfig(_PLUGIN_IDENTIFIER, {}),PluginOAuthClientConfig(_PLUGIN_IDENTIFIER, {"oc": "v"}),],@@ -359,12 +361,11 @@ class TestDatasourceProviderService:persist(sqlite_session, p)with (patch("services.data_source.provider_service.get_current_user", return_value=mock_user),- patch.object(- service,- "get_oauth_client",+ patch(+ "services.data_source.credential_adapters.PluginDatasourceOAuthClientResolver.resolve",return_value=oauth_client,),- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"tok": "plain"}),+ patch.object(service._codec, "decrypt", return_value={"tok": "plain"}),):service.get_datasource_credentials("t1", "prov", "org/plug")sqlite_session.expire_all()@@ -386,15 +387,13 @@ class TestDatasourceProviderService:persist(sqlite_session, p)with (patch("services.data_source.provider_service.get_current_user", return_value=mock_user),- patch("services.data_source.provider_service.OAuthHandler") as oauth_handler,- patch.object(- service,- "get_oauth_client",+ patch(+ "services.data_source.credential_adapters.PluginDatasourceOAuthClientResolver.resolve",return_value=PluginOAuthClientConfig("org/plug:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"oc": "v"}),),- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"tok": "plain"}),+ patch.object(service._codec, "decrypt", return_value={"tok": "plain"}),):- oauth_handler.return_value.refresh_credentials.side_effect = RuntimeError("token endpoint failed")+ self._oauth.return_value.refresh_credentials.side_effect = RuntimeError("token endpoint failed")with pytest.raises(ValueError, match="provider prov"):service.get_datasource_credentials("t1", "prov", "org/plug")@@ -404,7 +403,7 @@ class TestDatasourceProviderService:persist(sqlite_session, p)with (patch("services.data_source.provider_service.get_current_user", return_value=mock_user),- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"k": "plain"}),+ patch.object(service._codec, "decrypt", return_value={"k": "plain"}),):result = service.get_datasource_credentials("t1", "prov", "org/plug")assert result == {"k": "plain"}@@ -415,7 +414,7 @@ class TestDatasourceProviderService:persist(sqlite_session, p)with (patch("services.data_source.provider_service.get_current_user", return_value=mock_user),- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"k": "v"}),+ patch.object(service._codec, "decrypt", return_value={"k": "v"}),):result = service.get_datasource_credentials("t1", "prov", "org/plug", credential_id="cred-id")assert result == {"k": "v"}@@ -442,7 +441,7 @@ class TestDatasourceProviderService:super().__init__(**kwargs)opened.append(self)- def refresh(**_kwargs: object) -> tuple[dict[str, str], int]:+ def refresh(**_kwargs: object) -> RefreshedDatasourceCredential:assert openedassert all(not session.in_transaction() for session in opened)if concurrent_update:@@ -452,19 +451,19 @@ class TestDatasourceProviderService:.where(DatasourceProvider.id == row.id).values(encrypted_credentials={"token": "concurrent"}, expires_at=int(time.time()) + 3600))- return {"token": "refreshed"}, int(time.time()) + 3600+ return RefreshedDatasourceCredential({"token": "refreshed"}, int(time.time()) + 3600)- service = DatasourceProviderService(+ service = build_datasource_provider_service(credentials=SQLAlchemyDatasourceCredentialRepository(session_factory=sessionmaker(bind=sqlite_session.get_bind(), class_=ObservedSession)))monkeypatch.setattr(service_module, "get_current_user", lambda: mock_user)- monkeypatch.setattr(service, "_refresh_datasource_credentials", refresh)+ monkeypatch.setattr(service._refresher, "refresh", refresh)monkeypatch.setattr(- service,- "decrypt_datasource_provider_credentials",- lambda **kwargs: dict(kwargs["datasource_provider"].encrypted_credentials),+ service._codec,+ "decrypt",+ lambda record: dict(record.encrypted_credentials),)result = service.get_datasource_credentials("t1", "prov", "org/plug")assert result == {"token": "concurrent" if concurrent_update else "refreshed"}@@ -489,12 +488,11 @@ class TestDatasourceProviderService:persist(sqlite_session, p)with (patch("services.data_source.provider_service.get_current_user", return_value=mock_user),- patch.object(- service,- "get_oauth_client",+ patch(+ "services.data_source.credential_adapters.PluginDatasourceOAuthClientResolver.resolve",return_value=PluginOAuthClientConfig("org/plug:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"oc": "v"}),),- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"t": "plain"}),+ patch.object(service._codec, "decrypt", return_value={"t": "plain"}),):result = service.get_all_datasource_credentials_by_provider("t1", "prov", "org/plug")assert len(result) == 1@@ -518,11 +516,14 @@ class TestDatasourceProviderService:with (patch("services.data_source.provider_service.get_current_user", return_value=mock_user),patch.object(- service,- "_refresh_datasource_credentials",- side_effect=[ValueError("refresh failed"), ({"t": "enc"}, 9999)],+ service._refresher,+ "refresh",+ side_effect=[+ DatasourceCredentialRefreshError("failed-cred"),+ RefreshedDatasourceCredential({"t": "plain"}, 9999),+ ],) as refresh_credentials,- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"t": "plain"}),+ patch.object(service._codec, "decrypt", return_value={"t": "plain"}),):result = service.get_all_datasource_credentials_by_provider("t1", "prov", "org/plug")assert result == [{"t": "plain"}]@@ -536,8 +537,8 @@ class TestDatasourceProviderService:persist(sqlite_session, p)with (patch("services.data_source.provider_service.get_current_user", return_value=mock_user),- patch.object(service, "_refresh_datasource_credentials") as refresh_credentials,- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"t": "plain"}),+ patch.object(service._refresher, "refresh") as refresh_credentials,+ patch.object(service._codec, "decrypt", return_value={"t": "plain"}),):result = service.get_all_datasource_credentials_by_provider("t1", "prov", "org/plug")assert result == [{"t": "plain"}]diff --git a/api/tests/unit_tests/services/data_source/test_website_service.py b/api/tests/unit_tests/services/data_source/test_website_service.pyindex cde5d32a8ab..2be7df041ff 100644--- a/api/tests/unit_tests/services/data_source/test_website_service.py+++ b/api/tests/unit_tests/services/data_source/test_website_service.py@@ -22,6 +22,7 @@ from services.data_source.website_service import (WebsiteCrawlStatusApiRequest,WebsiteService,)+from tests.helpers.unit.credential_refresh import build_datasource_provider_service@dataclass(frozen=True)@@ -112,11 +113,7 @@ def test_get_credentials_and_config_selects_plugin_id_and_key_firecrawl(monkeypaservice_instance.get_datasource_credentials.return_value = {"firecrawl_api_key": "k", "base_url": "b"}monkeypatch.setattr(website_service_module, "DatasourceProviderService", MagicMock(return_value=service_instance))- api_key, config = WebsiteService(- providers=website_service_module.DatasourceProviderService(- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)- )- )._get_credentials_and_config("tenant-1", "firecrawl")+ api_key, config = WebsiteService(providers=service_instance)._get_credentials_and_config("tenant-1", "firecrawl")assert api_key == "k"assert config["base_url"] == "b"@@ -141,11 +138,7 @@ def test_get_credentials_and_config_selects_plugin_id_and_key_api_key(service_instance.get_datasource_credentials.return_value = {"api_key": "enc-key", "base_url": "b"}monkeypatch.setattr(website_service_module, "DatasourceProviderService", MagicMock(return_value=service_instance))- api_key, config = WebsiteService(- providers=website_service_module.DatasourceProviderService(- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)- )- )._get_credentials_and_config("tenant-1", provider)+ api_key, config = WebsiteService(providers=service_instance)._get_credentials_and_config("tenant-1", provider)assert api_key == "enc-key"assert config["base_url"] == "b"@@ -159,7 +152,7 @@ def test_get_credentials_and_config_selects_plugin_id_and_key_api_key(def test_get_credentials_and_config_rejects_invalid_provider() -> None:with pytest.raises(ValueError, match="Invalid provider"):WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)))._get_credentials_and_config("tenant-1", "unknown")@@ -187,11 +180,7 @@ def test_get_credentials_and_config_hits_unreachable_guard_branch(monkeypatch: pmonkeypatch.setattr(website_service_module, "DatasourceProviderService", MagicMock(return_value=service_instance))with pytest.raises(ValueError, match="Invalid provider"):- WebsiteService(- providers=website_service_module.DatasourceProviderService(- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)- )- )._get_credentials_and_config("tenant-1", FlakyProvider()) # type: ignore[arg-type]+ WebsiteService(providers=service_instance)._get_credentials_and_config("tenant-1", FlakyProvider()) # type: ignore[arg-type]def test_get_decrypted_api_key_requires_api_key(monkeypatch: pytest.MonkeyPatch) -> None:@@ -222,7 +211,7 @@ def test_crawl_url_dispatches_by_provider(monkeypatch: pytest.MonkeyPatch) -> Nomonkeypatch.setattr(WebsiteService, "_crawl_with_firecrawl", firecrawl_mock)result = WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).crawl_url(api_request)@@ -247,7 +236,7 @@ def test_crawl_url_dispatches_other_providers(monkeypatch: pytest.MonkeyPatch, pmonkeypatch.setattr(WebsiteService, method_name, impl_mock)assert WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).crawl_url(api_request) == {"status": "active"}@@ -260,7 +249,7 @@ def test_crawl_url_rejects_invalid_provider(monkeypatch: pytest.MonkeyPatch) ->with pytest.raises(ValueError, match="Invalid provider"):WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).crawl_url(api_request)@@ -436,7 +425,7 @@ def test_get_crawl_status_dispatches(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr(WebsiteService, "_get_firecrawl_status", firecrawl_status)result = WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).get_crawl_status("job-1", "firecrawl")@@ -446,7 +435,7 @@ def test_get_crawl_status_dispatches(monkeypatch: pytest.MonkeyPatch) -> None:watercrawl_status = MagicMock(return_value={"status": "active", "job_id": "w"})monkeypatch.setattr(WebsiteService, "_get_watercrawl_status", watercrawl_status)assert WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).get_crawl_status("job-2", "watercrawl") == {"status": "active", "job_id": "w"}@@ -455,7 +444,7 @@ def test_get_crawl_status_dispatches(monkeypatch: pytest.MonkeyPatch) -> None:jinareader_status = MagicMock(return_value={"status": "active", "job_id": "j"})monkeypatch.setattr(WebsiteService, "_get_jinareader_status", jinareader_status)assert WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).get_crawl_status("job-3", "jinareader") == {"status": "active", "job_id": "j"}@@ -466,7 +455,7 @@ def test_get_crawl_status_typed_rejects_invalid_provider(monkeypatch: pytest.Monmonkeypatch.setattr(WebsiteService, "_get_credentials_and_config", MagicMock(return_value=("k", {})))with pytest.raises(ValueError, match="Invalid provider"):WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).get_crawl_status_typed(WebsiteCrawlStatusApiRequest(provider="bad", job_id="j"))@@ -578,7 +567,7 @@ def test_get_jinareader_status_completed_formats_processed_items(monkeypatch: pydef test_get_crawl_url_data_dispatches_invalid_provider() -> None:with pytest.raises(ValueError, match="Invalid provider"):WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).get_crawl_url_data("job-1", "bad", "https://example.com", "tenant-1")@@ -590,7 +579,7 @@ def test_get_crawl_url_data_hits_invalid_provider_branch_when_credentials_stubbemonkeypatch.setattr(WebsiteService, "_get_credentials_and_config", MagicMock(return_value=("k", {})))with pytest.raises(ValueError, match="Invalid provider"):WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).get_crawl_url_data("job-1", object(), "u", "tenant-1") # type: ignore[arg-type]@@ -610,7 +599,7 @@ def test_get_crawl_url_data_dispatches(monkeypatch: pytest.MonkeyPatch, providermonkeypatch.setattr(WebsiteService, method_name, impl_mock)result = WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).get_crawl_url_data("job-1", provider, "u", "tenant-1")@@ -734,7 +723,7 @@ def test_get_scrape_url_data_dispatches_and_rejects_invalid_provider(monkeypatchscrape_mock = MagicMock(return_value={"data": "x"})monkeypatch.setattr(WebsiteService, "_scrape_with_firecrawl", scrape_mock)assert WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).get_scrape_url_data("firecrawl", "u", "tenant-1", True) == {"data": "x"}@@ -743,7 +732,7 @@ def test_get_scrape_url_data_dispatches_and_rejects_invalid_provider(monkeypatchwatercrawl_mock = MagicMock(return_value={"data": "y"})monkeypatch.setattr(WebsiteService, "_scrape_with_watercrawl", watercrawl_mock)assert WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).get_scrape_url_data("watercrawl", "u", "tenant-1", False) == {"data": "y"}@@ -751,7 +740,7 @@ def test_get_scrape_url_data_dispatches_and_rejects_invalid_provider(monkeypatchwith pytest.raises(ValueError, match="Invalid provider"):WebsiteService(- providers=website_service_module.DatasourceProviderService(+ providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True))).get_scrape_url_data("jinareader", "u", "tenant-1", True)diff --git a/api/tests/unit_tests/services/knowledge/indexing/adapters/test_estimate.py b/api/tests/unit_tests/services/knowledge/indexing/adapters/test_estimate.pyindex 16efd0c6953..b52b1ceb845 100644--- a/api/tests/unit_tests/services/knowledge/indexing/adapters/test_estimate.py+++ b/api/tests/unit_tests/services/knowledge/indexing/adapters/test_estimate.py@@ -17,6 +17,7 @@ from extensions.storage.storage_type import StorageTypefrom models.dataset import Dataset, DatasetProcessRulefrom models.enums import CreatorUserRole, ProcessRuleModefrom models.model import UploadFile+from services.credentials.refresh import CredentialRefreshUnavailableErrorfrom services.knowledge.indexing.adapters.estimate import (IndexingEstimateAdapter,SQLAlchemyProcessRuleReader,@@ -198,6 +199,7 @@ def test_estimate_selects_dataset_embedding_and_rejects_foreign_tenant(@pytest.mark.parametrize(("processor_error", "expected_error"),[+ (CredentialRefreshUnavailableError(), CredentialRefreshUnavailableError),(LLMBadRequestError("provider unavailable"), IndexingEstimateProviderUnavailableError),(ProviderTokenNotInitError("provider unavailable"), IndexingEstimateProviderUnavailableError),(PluginDaemonClientSideError("provider unavailable"), IndexingEstimateProviderUnavailableError),@@ -212,7 +214,7 @@ def test_estimate_translates_infrastructure_errors(adapter = IndexingEstimateAdapter(session_factory=sqlite_session_factory)with (patch("services.knowledge.indexing.adapters.estimate.IndexProcessorFactory", side_effect=processor_error),- pytest.raises(expected_error, match="provider unavailable|processor failed"),+ pytest.raises(expected_error, match="provider unavailable|processor failed|temporarily unavailable"),):adapter.run("workspace-1",diff --git a/api/tests/unit_tests/services/knowledge/indexing/adapters/test_sources.py b/api/tests/unit_tests/services/knowledge/indexing/adapters/test_sources.pyindex 1ee3038b1b3..da387056de5 100644--- a/api/tests/unit_tests/services/knowledge/indexing/adapters/test_sources.py+++ b/api/tests/unit_tests/services/knowledge/indexing/adapters/test_sources.py@@ -9,6 +9,7 @@ from core.rag.extractor.entity.datasource_type import DatasourceTypefrom core.rag.extractor.entity.extract_setting import UploadFileExtractionInputfrom core.rag.extractor.notion_extractor import NotionExtractorfrom extensions.application_services.data_sources import build_data_source_credentials+from services.credentials.refresh import CredentialRefreshUnavailableErrorfrom services.data_source.credential_gateway import (DatasourceCredentialNotFoundError,DatasourceCredentialRefreshError,@@ -186,11 +187,21 @@ def test_stored_notion_resolution_uses_trusted_document_owner_chain() -> None:("credential-1", {"integration_secret": "saved-secret"}, None, "saved-secret"),("deleted-credential", {}, DatasourceCredentialNotFoundError(), "environment-secret"),("credential-1", {}, DatasourceCredentialRefreshError("credential-1"), "environment-secret"),+ ("credential-1", {}, CredentialRefreshUnavailableError(), "environment-secret"),("credential-1", {}, None, "environment-secret"),("credential-1", {"integration_secret": ""}, None, "environment-secret"),("credential-1", {"integration_secret": 42}, None, "environment-secret"),],- ids=["legacy", "saved", "deleted", "refresh-failed", "missing-token", "empty-token", "invalid-token"],+ ids=[+ "legacy",+ "saved",+ "deleted",+ "refresh-failed",+ "refresh-unavailable",+ "missing-token",+ "empty-token",+ "invalid-token",+ ],)@pytest.mark.parametrize("integration_token", ["environment-secret", None, ""])def test_stored_notion_preview_and_indexing_use_same_integration(@@ -234,7 +245,12 @@ def test_stored_notion_preview_and_indexing_use_same_integration(),)- if expected_token == "environment-secret" and not integration_token:+ if isinstance(error, CredentialRefreshUnavailableError) and not integration_token:+ with pytest.raises(CredentialRefreshUnavailableError):+ resolver.resolve(source)+ with pytest.raises(CredentialRefreshUnavailableError):+ indexing_extractor()+ elif expected_token == "environment-secret" and not integration_token:with pytest.raises(SourceCredentialUnavailableError):resolver.resolve(source)with pytest.raises(ValueError, match="Must specify `integration_token`"):diff --git a/api/tests/unit_tests/services/knowledge/indexing/test_estimate.py b/api/tests/unit_tests/services/knowledge/indexing/test_estimate.pyindex 9cb7cc05066..de552f29827 100644--- a/api/tests/unit_tests/services/knowledge/indexing/test_estimate.py+++ b/api/tests/unit_tests/services/knowledge/indexing/test_estimate.py@@ -8,6 +8,7 @@ from core.entities.knowledge_entities import IndexingEstimatefrom core.rag.extractor.entity.datasource_type import DatasourceTypefrom core.rag.extractor.entity.extract_setting import ExtractSetting, UploadFileExtractionInputfrom machinery.context import RequestContext+from services.credentials.refresh import CredentialRefreshUnavailableErrorfrom services.data_source.credential_gateway import DatasourceCredentialNotFoundErrorfrom services.data_source.entities.notion_import import NotionPageTypefrom services.knowledge.dataset_access import AccessibleDataset, DatasetAccess, DatasetAccessDeniedError@@ -333,6 +334,7 @@ def test_estimate_rejects_missing_document(fixture: ServiceFixture) -> None:@pytest.mark.parametrize(("source_error", "expected_error"),[+ (CredentialRefreshUnavailableError(), CredentialRefreshUnavailableError),(DatasourceCredentialNotFoundError(), IndexingEstimateCredentialUnavailableError),(IndexingInputSourceError("missing source"), EstimateSourceNotFoundError),(UnsupportedStoredSourceError("unknown"), UnsupportedEstimateSourceError),diff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline.pyindex 88944418c83..c397af2789b 100644--- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline.py+++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline.py@@ -46,6 +46,7 @@ from services.errors.rag_pipeline import RagPipelineResourceNotFoundErrorfrom services.rag_pipeline import rag_pipeline as rag_pipeline_modulefrom services.rag_pipeline.rag_pipeline import RagPipelineServicefrom services.workflow_ref_service import WorkflowRef+from tests.helpers.unit.credential_refresh import build_datasource_provider_servicedef _make_service() -> RagPipelineService:@@ -776,7 +777,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="website_crawl",is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -861,7 +862,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="online_document",is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -1043,9 +1044,7 @@ class TestRagPipelineServiceCases:"d1",True,credential_query=mock.create_autospec(CredentialQuery, instance=True),- datasource_providers=rag_pipeline_module.DatasourceProviderService(- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)- ),+ datasource_providers=mock_provider_service,)# 3. Assertions@@ -1352,7 +1351,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="online_document",is_published=False,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -1405,7 +1404,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -1462,7 +1461,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type=DatasourceProviderType.ONLINE_DRIVE,is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -1696,7 +1695,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="online_document",is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -1755,7 +1754,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="online_document",is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -1816,7 +1815,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="online_document",is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -1891,7 +1890,7 @@ class TestRagPipelineServiceCases:"d1",True,credential_query=mock.create_autospec(CredentialQuery, instance=True),- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -1995,7 +1994,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="website_crawl",is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -2181,7 +2180,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="website_crawl",is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -2235,7 +2234,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="online_drive",is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -2290,7 +2289,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="online_document",is_published=False,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -2452,7 +2451,7 @@ class TestRagPipelineServiceCases:"d1",True,credential_query=mock.create_autospec(CredentialQuery, instance=True),- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -2489,7 +2488,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="online_document",is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -2510,7 +2509,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="online_document",is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -2563,7 +2562,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="online_document",is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -2610,7 +2609,7 @@ class TestRagPipelineServiceCases:account=_make_account(),datasource_type="online_document",is_published=True,- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -2652,7 +2651,7 @@ class TestRagPipelineServiceCases:"d1",False,credential_query=mock.create_autospec(CredentialQuery, instance=True),- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)@@ -2701,7 +2700,7 @@ class TestRagPipelineServiceCases:"d1",True,credential_query=mock.create_autospec(CredentialQuery, instance=True),- datasource_providers=rag_pipeline_module.DatasourceProviderService(+ datasource_providers=build_datasource_provider_service(credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)),)diff --git a/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_layer.py b/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_layer.pyindex 2fe15eb5f03..e0aafb243fc 100644--- a/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_layer.py+++ b/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_layer.py@@ -17,6 +17,11 @@ from tests.local.dify_agent.module_support import invoke_native_tool[(200, {"observation": "completed", "messages": [], "metadata": {}}, "completed"),(429, {"code": "rate_limit", "message": "busy"}, "temporarily unavailable"),+ (+ 503,+ {"code": "service_unavailable", "message": "Credential refresh is temporarily unavailable. Please retry."},+ "temporarily unavailable",+ ),(404, {"code": "app_not_found", "message": "missing"}, "app context no longer exists"),(403, {"code": "app_tenant_mismatch", "message": "invalid"}, "app context is invalid"),(422, {"code": "agent_tool_credential_invalid", "message": "invalid"}, "check your tool provider credentials"),