Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(oauth): serialize and persist provider credential refreshes (#32195)

Peter Pan5907db83b0d0
diff --git a/.github/workflows/knowledge-fs-ci.yml b/.github/workflows/knowledge-fs-ci.yml
index 6883917c087..0b2757f47a8 100644
--- a/.github/workflows/knowledge-fs-ci.yml
+++ b/.github/workflows/knowledge-fs-ci.yml
@@ -57,6 +57,7 @@ jobs:
- 'api/.env.example'
- 'api/app_factory.py'
- 'api/commands/__init__.py'
+ - 'api/controllers/common/errors.py'
- 'api/controllers/console/__init__.py'
- 'api/controllers/console/workspace/rbac/**'
- 'api/controllers/inner_api/__init__.py'
@@ -84,7 +85,10 @@ jobs:
- 'api/tests/unit_tests/dev/test_generate_knowledge_fs_contract.py'
- 'api/tests/unit_tests/extensions/test_ext_celery.py'
- 'api/tests/unit_tests/models/test_agent_config_entities.py'
+ - 'api/tests/unit_tests/controllers/common/test_errors.py'
+ - 'api/tests/unit_tests/controllers/console/datasets/test_data_source.py'
- 'api/tests/unit_tests/controllers/console/workspace/rbac/test_roles.py'
+ - 'api/tests/unit_tests/controllers/inner_api/agent/test_tools.py'
- 'api/tests/unit_tests/core/agent/test_base_agent_runner.py'
- 'api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py'
- 'api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py'
@@ -196,6 +200,7 @@ jobs:
production_touchpoints=(
api/app_factory.py
api/commands/__init__.py
+ api/controllers/common/errors.py
api/controllers/console/__init__.py
api/controllers/console/datasets/datasets.py
api/controllers/console/workspace/rbac/__init__.py
@@ -257,9 +262,12 @@ jobs:
test_touchpoints=(
api/tests/unit_tests/extensions/test_ext_celery.py
api/tests/unit_tests/models/test_agent_config_entities.py
+ api/tests/unit_tests/controllers/common/test_errors.py
+ api/tests/unit_tests/controllers/console/datasets/test_data_source.py
api/tests/unit_tests/controllers/console/datasets/test_datasets.py
api/tests/unit_tests/controllers/console/test_wraps.py
api/tests/unit_tests/controllers/console/workspace/rbac/test_roles.py
+ api/tests/unit_tests/controllers/inner_api/agent/test_tools.py
api/tests/unit_tests/core/agent/test_base_agent_runner.py
api/tests/unit_tests/core/app/apps/agent_app/test_runtime_request_builder.py
api/tests/unit_tests/core/workflow/nodes/agent_v2/test_runtime_request_builder.py
diff --git a/api/.importlinter b/api/.importlinter
index 6f7a91ab446..789f8fec427 100644
--- a/api/.importlinter
+++ b/api/.importlinter
@@ -313,6 +313,35 @@ forbidden_modules =
flask
allow_indirect_imports = True
+[importlinter:contract:tool-credential-application-boundary]
+name = Runtime credential refresh policies are framework and persistence neutral
+type = forbidden
+source_modules =
+ services.credentials.tool_runtime
+ services.credentials.refresh
+forbidden_modules =
+ configs
+ controllers
+ core
+ extensions
+ flask
+ models
+ repositories
+ sqlalchemy
+ tasks
+ werkzeug
+
+[importlinter:contract:tool-credential-adapter-boundary]
+name = Tool runtime credential persistence is reached only through composition
+type = forbidden
+source_modules =
+ core.tools.tool_manager
+forbidden_modules =
+ repositories
+ services.credentials.tool_runtime_adapters
+ services.credentials.redis_refresh_lock
+allow_indirect_imports = True
+
[importlinter:contract:credential-query-implementation-boundary]
name = Credential query implementations are wired only by composition or other repositories
type = forbidden
diff --git a/api/controllers/common/errors.py b/api/controllers/common/errors.py
index 987023931ae..a8fdaf9810b 100644
--- a/api/controllers/common/errors.py
+++ b/api/controllers/common/errors.py
@@ -7,6 +7,7 @@ from flask import current_app, got_request_exception
from core.logging.context import get_request_id
from libs.exception import BaseHTTPException
from libs.external_api import ExternalApi
+from services.credentials.refresh import CredentialRefreshUnavailableError
from services.errors.base import NoPermissionError
@@ -181,3 +182,20 @@ def register_permission_error_handler(api: ExternalApi) -> None:
# Flask-RESTX picks the first matching handler, including the generic Exception handler.
cast(OrderedDict[type[Exception], object], api.error_handlers).move_to_end(NoPermissionError, last=False)
api.document_error_handler(NoPermissionError, status=HTTPStatus.BAD_REQUEST, code=InvalidArgumentError.error_code)
+
+
+def register_credential_refresh_error_handler(api: ExternalApi) -> None:
+ """Give temporary refresh failures the same retryable response on each API surface."""
+
+ @api.errorhandler(CredentialRefreshUnavailableError)
+ def handle_refresh_unavailable(error: CredentialRefreshUnavailableError) -> tuple[dict[str, str | int], int]:
+ got_request_exception.send(current_app, exception=error)
+ status = HTTPStatus.SERVICE_UNAVAILABLE
+ return {"code": "service_unavailable", "message": str(error), "status": status.value}, status.value
+
+ handlers: object = api.error_handlers
+ assert isinstance(handlers, OrderedDict)
+ handlers.move_to_end(CredentialRefreshUnavailableError, last=False)
+ api.document_error_handler(
+ CredentialRefreshUnavailableError, status=HTTPStatus.SERVICE_UNAVAILABLE, code="service_unavailable"
+ )
diff --git a/api/controllers/console/__init__.py b/api/controllers/console/__init__.py
index 9af9886a7be..96253ec6c6f 100644
--- a/api/controllers/console/__init__.py
+++ b/api/controllers/console/__init__.py
@@ -13,6 +13,7 @@ from controllers.common.errors import (
InvalidArgumentError,
NotFoundError,
UnauthorizedError,
+ register_credential_refresh_error_handler,
register_permission_error_handler,
)
from controllers.console.error import NotInitValidateError, NotSetupError, UnauthorizedAndForceLogout
@@ -36,6 +37,7 @@ api = ExternalApi(
description="Console management APIs for app configuration, monitoring, and administration",
)
register_permission_error_handler(api)
+register_credential_refresh_error_handler(api)
# Errors the Console client handles centrally (bootstrap redirects, forced logout, malformed
# input); any operation may answer with them, so no operation lists them. The shared
# ``not_found``, ``unauthorized`` and ``internal_server_error`` codes are the ones Werkzeug's own
diff --git a/api/controllers/inner_api/__init__.py b/api/controllers/inner_api/__init__.py
index 322066d6adf..d06c8be588c 100644
--- a/api/controllers/inner_api/__init__.py
+++ b/api/controllers/inner_api/__init__.py
@@ -3,6 +3,7 @@ from types import ModuleType
from flask import Blueprint
from flask_restx import Namespace
+from controllers.common.errors import register_credential_refresh_error_handler
from libs.external_api import ExternalApi
bp = Blueprint("inner_api", __name__, url_prefix="/inner/api")
@@ -22,6 +23,7 @@ api = ExternalApi(
},
security="InnerApiKey",
)
+register_credential_refresh_error_handler(api)
# Create namespace
inner_api_ns = Namespace("inner_api", description="Internal API operations", path="/")
diff --git a/api/core/helper/provider_cache.py b/api/core/helper/provider_cache.py
index 8e85d444d2c..d4ad7b2bb99 100644
--- a/api/core/helper/provider_cache.py
+++ b/api/core/helper/provider_cache.py
@@ -59,8 +59,35 @@ class SingletonProviderCredentialsCache(ProviderCredentialsCache):
class ToolProviderCredentialsCache(ProviderCredentialsCache):
"""Cache for tool provider credentials"""
- def __init__(self, tenant_id: str, provider: str, credential_id: str):
+ def __init__(self, tenant_id: str, provider: str, credential_id: str, *, fingerprint: str | None = None):
+ """A fingerprint selects one snapshot-checked entry; None retains the legacy cache."""
super().__init__(tenant_id=tenant_id, provider=provider, credential_id=credential_id)
+ self._legacy_cache_key = self.cache_key
+ self._fingerprint = fingerprint
+ if fingerprint is not None:
+ self.cache_key += ":snapshot"
+
+ @override
+ def get(self) -> dict[str, Any] | None:
+ cached = super().get()
+ if self._fingerprint is None or cached is None:
+ return cached
+ credentials = cached.get("credentials")
+ if cached.get("fingerprint") == self._fingerprint and isinstance(credentials, dict):
+ return credentials
+ return None
+
+ @override
+ def set(self, config: dict[str, Any]) -> None:
+ if self._fingerprint is not None:
+ config = {"fingerprint": self._fingerprint, "credentials": config}
+ super().set(config)
+
+ @override
+ def delete(self) -> None:
+ # Separate commands also work when Redis Cluster puts the entries in different slots.
+ redis_client.delete(self._legacy_cache_key)
+ redis_client.delete(f"{self._legacy_cache_key}:snapshot")
@override
def _generate_cache_key(self, **kwargs: Any) -> str:
diff --git a/api/core/tools/tool_manager.py b/api/core/tools/tool_manager.py
index 20f4c5e67bd..b97a68ad914 100644
--- a/api/core/tools/tool_manager.py
+++ b/api/core/tools/tool_manager.py
@@ -1,11 +1,9 @@
-import json
import logging
import mimetypes
-import time
from collections.abc import Generator, Mapping
from os import listdir, path
from threading import Lock
-from typing import TYPE_CHECKING, Any, Literal, Protocol, cast
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Protocol, cast
import sqlalchemy as sa
from pydantic import TypeAdapter
@@ -18,10 +16,9 @@ import contexts
from configs import dify_config
from core.agent.entities import AgentToolEntity
from core.app.entities.app_invoke_entities import InvokeFrom
-from core.entities import PluginCredentialType
from core.helper.module_import_helper import load_single_subclass_from_source
from core.helper.position_helper import is_filtered
-from core.helper.provider_cache import ToolProviderCredentialsCache
+from core.plugin.entities.plugin_daemon import CredentialType
from core.plugin.impl.exc import PluginDaemonNotFoundError, PluginNotFoundError
from core.plugin.impl.tool import PluginToolManager
from core.tools.__base.tool import Tool
@@ -43,14 +40,14 @@ from core.tools.entities.tool_entities import (
ToolProviderType,
emoji_icon_adapter,
)
-from core.tools.errors import ToolProviderCredentialValidationError, ToolProviderNotFoundError
+from core.tools.errors import ToolProviderNotFoundError
from core.tools.mcp_tool.provider import MCPToolProviderController
from core.tools.mcp_tool.tool import MCPTool
from core.tools.plugin_tool.provider import PluginToolProviderController
from core.tools.plugin_tool.tool import PluginTool
from core.tools.tool_label_manager import ToolLabelManager
from core.tools.utils.configuration import ToolParameterConfigurationManager
-from core.tools.utils.encryption import create_provider_encrypter, create_tool_provider_encrypter
+from core.tools.utils.encryption import create_tool_provider_encrypter
from core.tools.utils.uuid_utils import is_valid_uuid
from core.tools.workflow_as_tool.provider import WorkflowToolProviderController
from core.tools.workflow_as_tool.tool import WorkflowTool
@@ -93,12 +90,29 @@ class WorkflowToolRuntimeSpec(Protocol):
def credential_id(self) -> str | None: ...
+class ToolCredentialResolver(Protocol):
+ def __call__(
+ self,
+ *,
+ controller: BuiltinToolProviderController,
+ provider_id: str,
+ workspace_id: str,
+ credential_id: str | None,
+ ) -> tuple[Mapping[str, object], CredentialType]: ...
+
+
class ToolManager:
_builtin_provider_lock = Lock()
_hardcoded_providers: dict[str, BuiltinToolProviderController] = {}
_external_hardcoded_provider_classes: list[type[BuiltinToolProviderController]] = []
_builtin_providers_loaded = False
_builtin_tools_labels: dict[str, I18nObject | None] = {}
+ _credential_resolver: ClassVar[ToolCredentialResolver | None] = None
+
+ @classmethod
+ def configure_credential_resolver(cls, resolver: ToolCredentialResolver) -> None:
+ """Register stateless composition; it must not capture an app, database or provider."""
+ cls._credential_resolver = resolver
@classmethod
def get_hardcoded_provider(cls, provider: str) -> BuiltinToolProviderController:
@@ -229,141 +243,26 @@ class ToolManager:
tool_invoke_from=tool_invoke_from,
)
)
- builtin_provider = None
- if isinstance(provider_controller, PluginToolProviderController):
- provider_id_entity = ToolProviderID(provider_id)
- if is_valid_uuid(credential_id):
- try:
- builtin_provider_stmt = select(BuiltinToolProvider).where(
- BuiltinToolProvider.tenant_id == tenant_id,
- BuiltinToolProvider.id == credential_id,
- )
- builtin_provider = db.session.scalar(builtin_provider_stmt)
- except Exception as e:
- builtin_provider = None
- logger.info("Error getting builtin provider %s:%s", credential_id, e, exc_info=True)
- if builtin_provider is None:
- raise ToolProviderCredentialValidationError(
- f"Tool credential {credential_id} has been deleted. "
- "Select or authorize another credential."
- )
-
- if builtin_provider is None:
- with Session(db.engine) as session:
- builtin_provider = session.scalar(
- sa.select(BuiltinToolProvider)
- .where(
- BuiltinToolProvider.tenant_id == tenant_id,
- (BuiltinToolProvider.provider == str(provider_id_entity))
- | (BuiltinToolProvider.provider == provider_id_entity.provider_name),
- )
- .order_by(BuiltinToolProvider.is_default.desc(), BuiltinToolProvider.created_at.asc())
- )
- if builtin_provider is None:
- raise ToolProviderCredentialValidationError(
- f"No workspace credential is configured for tool provider {provider_id}. "
- "Authorize the provider or select a credential."
- )
- else:
- builtin_provider = db.session.scalar(
- select(BuiltinToolProvider)
- .where(
- BuiltinToolProvider.tenant_id == tenant_id, (BuiltinToolProvider.provider == provider_id)
- )
- .order_by(BuiltinToolProvider.is_default.desc(), BuiltinToolProvider.created_at.asc())
- .limit(1)
- )
-
- if builtin_provider is None:
- raise ToolProviderCredentialValidationError(
- f"No credential is configured for built-in tool provider {provider_id}. "
- "Authorize the provider or select a credential."
- )
-
- from core.helper.credential_utils import runtime_check_credential_policy_compliance
-
- runtime_check_credential_policy_compliance(
- credential_id=builtin_provider.id,
- provider=provider_id,
- credential_type=PluginCredentialType.TOOL,
- check_existence=False,
+ selected_id = (
+ credential_id
+ if isinstance(provider_controller, PluginToolProviderController) and is_valid_uuid(credential_id)
+ else None
)
-
- encrypter, cache = create_provider_encrypter(
- tenant_id=tenant_id,
- config=[
- x.to_basic_provider_config()
- for x in provider_controller.get_credentials_schema_by_type(builtin_provider.credential_type)
- ],
- cache=ToolProviderCredentialsCache(
- tenant_id=tenant_id, provider=provider_id, credential_id=builtin_provider.id
- ),
+ if cls._credential_resolver is None:
+ raise RuntimeError("Tool credential resolver is not configured")
+ credentials, credential_type = cls._credential_resolver(
+ controller=provider_controller,
+ provider_id=provider_id,
+ workspace_id=tenant_id,
+ credential_id=selected_id,
)
- decrypted_credentials: Mapping[str, Any] = encrypter.decrypt(builtin_provider.credentials)
-
- if builtin_provider.expires_at != -1 and (builtin_provider.expires_at - 60) < int(time.time()):
- # TODO: circular import
- from core.plugin.impl.oauth import OAuthHandler
- from services.tools.builtin_tools_manage_service import BuiltinToolManageService
-
- tool_provider = ToolProviderID(provider_id)
- provider_name = tool_provider.provider_name
- redirect_uri = f"{dify_config.CONSOLE_API_URL}/console/api/oauth/plugin/{provider_id}/tool/callback"
- plugin_unique_identifier = (
- provider_controller.plugin_unique_identifier
- if isinstance(provider_controller, PluginToolProviderController)
- else None
- )
- system_credentials = BuiltinToolManageService.get_oauth_client(
- tenant_id,
- provider_id,
- provider_controller=(
- provider_controller
- if isinstance(provider_controller, PluginToolProviderController)
- else None
- ),
- )
- if system_credentials is not None:
- plugin_unique_identifier = system_credentials.plugin_unique_identifier
-
- oauth_handler = OAuthHandler()
- try:
- refreshed_credentials = oauth_handler.refresh_credentials(
- tenant_id=tenant_id,
- user_id=builtin_provider.user_id,
- plugin_id=tool_provider.plugin_id,
- provider=provider_name,
- redirect_uri=redirect_uri,
- system_credentials=(
- dict(system_credentials.credentials) if system_credentials is not None else {}
- ),
- plugin_unique_identifier=plugin_unique_identifier,
- credentials=decrypted_credentials,
- )
- except Exception as exc:
- logger.warning(
- "Failed to refresh OAuth credentials for tool provider %s", provider_id, exc_info=True
- )
- raise ToolProviderCredentialValidationError(
- f"OAuth credential for tool provider {provider_id} could not be refreshed. "
- "Reauthorize or select another credential."
- ) from exc
- # update the credentials
- builtin_provider.encrypted_credentials = json.dumps(
- encrypter.encrypt(refreshed_credentials.credentials)
- )
- builtin_provider.expires_at = refreshed_credentials.expires_at
- db.session.commit()
- decrypted_credentials = refreshed_credentials.credentials
- cache.delete()
-
return builtin_tool.fork_tool_runtime(
runtime=ToolRuntime(
tenant_id=tenant_id,
user_id=user_id,
- credentials=dict(decrypted_credentials),
- credential_type=builtin_provider.credential_type,
+ credentials=dict(credentials),
+ credential_type=credential_type,
runtime_parameters={},
invoke_from=invoke_from,
tool_invoke_from=tool_invoke_from,
diff --git a/api/extensions/application_services/data_sources.py b/api/extensions/application_services/data_sources.py
index 0818c635e47..a5a09a4ca7e 100644
--- a/api/extensions/application_services/data_sources.py
+++ b/api/extensions/application_services/data_sources.py
@@ -10,11 +10,13 @@ from core.datasource.datasource_manager import DatasourceManager
from core.helper.ssrf_proxy import ssrf_proxy
from core.plugin.impl.datasource import PluginDatasourceManager
from core.plugin.impl.oauth import OAuthHandler
+from extensions.ext_redis import redis_client
from repositories.data_source.api_key_auth_repository import SQLAlchemyDataSourceApiKeyAuthBindingRepository
from repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository
from repositories.data_source.oauth_binding_repository import SQLAlchemyDataSourceOAuthBindingRepository
from repositories.knowledge.dataset_repository import SQLAlchemyDatasetRepository
from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository
+from services.credentials.redis_refresh_lock import RedisCredentialRefreshLock
from services.data_source.auth.api_key_gateways import (
ProviderApiKeyAuthCredentialValidator,
TenantApiKeyAuthCredentialEncryptor,
@@ -69,10 +71,21 @@ def build_data_source_credentials(*, database_client: sessionmaker[Session]) ->
oauth_clients=PluginDatasourceOAuthClientResolver(configs=credentials, provider_manager=provider_manager),
oauth_handler=OAuthHandler(),
)
+ refresh_lock = RedisCredentialRefreshLock(client=redis_client)
return DataSourceCredentials(
- providers=DatasourceProviderService(credentials=credentials),
- actor=ActorAwareDatasourceCredentialGateway(credentials=credentials, codec=codec, refresher=refresher),
- stored=TrustedStoredDatasourceCredentialGateway(credentials=credentials, codec=codec, refresher=refresher),
+ providers=DatasourceProviderService(
+ credentials=credentials,
+ codec=codec,
+ refresher=refresher,
+ refresh_lock=refresh_lock,
+ provider_manager=provider_manager,
+ ),
+ actor=ActorAwareDatasourceCredentialGateway(
+ credentials=credentials, codec=codec, refresher=refresher, refresh_lock=refresh_lock
+ ),
+ stored=TrustedStoredDatasourceCredentialGateway(
+ credentials=credentials, codec=codec, refresher=refresher, refresh_lock=refresh_lock
+ ),
)
diff --git a/api/extensions/application_services/tools.py b/api/extensions/application_services/tools.py
new file mode 100644
index 00000000000..67d36fc91a4
--- /dev/null
+++ b/api/extensions/application_services/tools.py
@@ -0,0 +1,35 @@
+"""Compose provider-scoped tool credential resolution for workflow execution."""
+
+from collections.abc import Mapping
+
+from configs import dify_config
+from core.db.session_factory import get_session_maker
+from core.plugin.entities.plugin_daemon import CredentialType
+from core.tools.builtin_tool.provider import BuiltinToolProviderController
+from core.tools.plugin_tool.provider import PluginToolProviderController
+from extensions.ext_redis import redis_client
+from models.provider_ids import ToolProviderID
+from repositories.credentials.query_repository import CredentialQueryRepository
+from services.credentials.redis_refresh_lock import RedisCredentialRefreshLock
+from services.credentials.tool_runtime import ToolCredentialService
+from services.credentials.tool_runtime_adapters import ProviderToolCredentialRuntime
+
+
+def resolve_tool_credentials(
+ *, controller: BuiltinToolProviderController, provider_id: str, workspace_id: str, credential_id: str | None
+) -> tuple[Mapping[str, object], CredentialType]:
+ if isinstance(controller, PluginToolProviderController):
+ provider = ToolProviderID(provider_id)
+ providers = (str(provider), provider.provider_name)
+ else:
+ providers = (provider_id,)
+ runtime = ProviderToolCredentialRuntime(
+ controller=controller, provider_id=provider_id, console_api_url=dify_config.CONSOLE_API_URL
+ )
+ service = ToolCredentialService(
+ credentials=CredentialQueryRepository(session_factory=get_session_maker()),
+ runtime=runtime,
+ refresh_lock=RedisCredentialRefreshLock(client=redis_client),
+ providers=providers,
+ )
+ return runtime.resolve(service=service, workspace_id=workspace_id, credential_id=credential_id)
diff --git a/api/extensions/ext_application_services.py b/api/extensions/ext_application_services.py
index f55df0150ae..66d46a01180 100644
--- a/api/extensions/ext_application_services.py
+++ b/api/extensions/ext_application_services.py
@@ -866,10 +866,12 @@ def build_application_services(
def init_app(app: Flask) -> None:
+ from extensions.application_services.tools import resolve_tool_credentials
from extensions.ext_login import bind_account_loader
app.extensions["knowledge_fs_node_dependencies"] = build_knowledge_fs_node_dependencies
ToolManager.register_hardcoded_provider_class(KnowledgeFSProvider)
+ ToolManager.configure_credential_resolver(resolve_tool_credentials)
services = build_application_services(
database_client=get_session_maker(),
deployment_edition=dify_config.DEPLOYMENT_EDITION,
diff --git a/api/repositories/credentials/query_repository.py b/api/repositories/credentials/query_repository.py
index 8e4d582738a..89605c4e85d 100644
--- a/api/repositories/credentials/query_repository.py
+++ b/api/repositories/credentials/query_repository.py
@@ -3,9 +3,11 @@
from collections.abc import Sequence
from typing import override
-from sqlalchemy import and_, func, or_, select
+from sqlalchemy import and_, func, or_, select, update
+from sqlalchemy.engine import CursorResult
from sqlalchemy.orm import InstrumentedAttribute, Session, sessionmaker
+from libs.datetime_utils import naive_utc_now
from models.credential_permission import CredentialPermission, CredentialType
from models.enums import PermissionEnum
from models.oauth import DatasourceProvider
@@ -20,6 +22,7 @@ from services.credentials.query import (
ToolCredentialRecord,
TriggerSubscriptionRecord,
)
+from services.credentials.tool_runtime import ToolRuntimeCredential
def apply_credential_visibility_filter_for_actor(
@@ -66,6 +69,55 @@ class CredentialQueryRepository(CredentialQuery):
def __init__(self, *, session_factory: sessionmaker[Session]) -> None:
self._session_factory = session_factory
+ def get_tool_credential(
+ self, *, workspace_id: str, providers: tuple[str, ...], credential_id: str | None
+ ) -> ToolRuntimeCredential | None:
+ statement = select(BuiltinToolProvider).where(
+ BuiltinToolProvider.tenant_id == workspace_id,
+ BuiltinToolProvider.provider.in_(providers),
+ )
+ if credential_id is not None:
+ statement = statement.where(BuiltinToolProvider.id == credential_id)
+ with self._session_factory() as session:
+ record = session.scalar(
+ statement.order_by(BuiltinToolProvider.is_default.desc(), BuiltinToolProvider.created_at.asc()).limit(1)
+ )
+ if record is None:
+ return None
+ return ToolRuntimeCredential(
+ id=record.id,
+ workspace_id=workspace_id,
+ owner_id=record.user_id,
+ provider=record.provider,
+ credential_type=record.credential_type.value,
+ encrypted_credentials=record.encrypted_credentials,
+ expires_at=record.expires_at,
+ )
+
+ def update_tool_credential_if_unchanged(
+ self, *, record: ToolRuntimeCredential, encrypted_credentials: str, expires_at: int
+ ) -> bool:
+ with self._session_factory.begin() as session:
+ result = session.execute(
+ update(BuiltinToolProvider)
+ .where(
+ BuiltinToolProvider.id == record.id,
+ BuiltinToolProvider.tenant_id == record.workspace_id,
+ BuiltinToolProvider.provider == record.provider,
+ BuiltinToolProvider.user_id == record.owner_id,
+ BuiltinToolProvider.credential_type == record.credential_type,
+ BuiltinToolProvider.encrypted_credentials == record.encrypted_credentials,
+ BuiltinToolProvider.expires_at == record.expires_at,
+ )
+ .values(
+ encrypted_credentials=encrypted_credentials,
+ expires_at=expires_at,
+ updated_at=naive_utc_now(),
+ )
+ )
+ assert isinstance(result, CursorResult)
+ return result.rowcount == 1
+
@override
def list_models(self, *, workspace_id: str, provider: str, actor_id: str | None) -> Sequence[ModelCredentialRecord]:
names = [provider]
diff --git a/api/repositories/data_source/credential_repository.py b/api/repositories/data_source/credential_repository.py
index ae38825e118..6121cc3afb7 100644
--- a/api/repositories/data_source/credential_repository.py
+++ b/api/repositories/data_source/credential_repository.py
@@ -256,6 +256,7 @@ def _update_datasource_credentials_if_unchanged(
DatasourceProvider.provider == record.provider,
DatasourceProvider.plugin_id == record.plugin_id,
DatasourceProvider.user_id == record.owner_id,
+ DatasourceProvider.auth_type == record.auth_type,
credentials_match,
DatasourceProvider.expires_at == record.expires_at,
)
diff --git a/api/services/agent_tool_inner_service.py b/api/services/agent_tool_inner_service.py
index 213e9ace816..085f37d471c 100644
--- a/api/services/agent_tool_inner_service.py
+++ b/api/services/agent_tool_inner_service.py
@@ -34,6 +34,7 @@ from core.tools.tool_engine import ToolEngine
from core.tools.tool_manager import ToolManager
from core.tools.utils.message_transformer import ToolFileMessageTransformer
from models.model import App
+from services.credentials.refresh import CredentialRefreshUnavailableError
from services.entities.agent_tool_inner import AgentToolInvokeRequest, AgentToolInvokeResponse
from services.errors.agent_tool_inner import AgentToolInnerServiceError
from services.workflow_run_agg import WorkflowRunAgg
@@ -106,6 +107,8 @@ class AgentToolInnerService:
conversation_id=request.caller.conversation_id,
)
)
+ except CredentialRefreshUnavailableError:
+ raise
except ToolProviderNotFoundError as exc:
raise AgentToolInnerServiceError(
error_code="agent_tool_declaration_not_found",
diff --git a/api/services/credentials/redis_refresh_lock.py b/api/services/credentials/redis_refresh_lock.py
new file mode 100644
index 00000000000..627af38219e
--- /dev/null
+++ b/api/services/credentials/redis_refresh_lock.py
@@ -0,0 +1,76 @@
+"""Redis leases for OAuth refreshes that can outlive a fixed lock timeout."""
+
+import logging
+from collections.abc import Generator
+from contextlib import contextmanager
+from threading import Event, Thread
+from typing import Literal
+
+from redis.exceptions import LockError, RedisError
+from redis.lock import Lock
+
+from extensions.ext_redis import RedisClientWrapper
+from services.credentials.refresh import CredentialRefreshLease, CredentialRefreshUnavailableError
+
+logger = logging.getLogger(__name__)
+
+_LEASE_SECONDS = 60
+_WAIT_SECONDS = 30
+_RENEW_SECONDS = 20
+
+
+class _RefreshLease:
+ def __init__(self, lock: Lock, lost: Event) -> None:
+ self._lock = lock
+ self._lost = lost
+
+ def ensure_owned(self) -> None:
+ if self._lost.is_set():
+ raise CredentialRefreshUnavailableError()
+ try:
+ self._lock.reacquire()
+ except (LockError, RedisError) as exc:
+ self._lost.set()
+ raise CredentialRefreshUnavailableError() from exc
+
+
+class RedisCredentialRefreshLock:
+ def __init__(self, *, client: RedisClientWrapper) -> None:
+ self._client = client
+
+ @contextmanager
+ def acquire(
+ self, *, credential_type: Literal["tool", "datasource"], workspace_id: str, credential_id: str
+ ) -> Generator[CredentialRefreshLease]:
+ name = f"oauth:credential-refresh:{credential_type}:{workspace_id}:{credential_id}"
+ lock = self._client.lock(name, timeout=_LEASE_SECONDS, blocking_timeout=_WAIT_SECONDS, thread_local=False)
+ try:
+ if not lock.acquire():
+ raise CredentialRefreshUnavailableError()
+ except (LockError, RedisError) as exc:
+ raise CredentialRefreshUnavailableError() from exc
+
+ stopped, lost = Event(), Event()
+ lease = _RefreshLease(lock, lost)
+
+ def renew() -> None:
+ while not stopped.wait(_RENEW_SECONDS):
+ try:
+ lease.ensure_owned()
+ except Exception:
+ lost.set()
+ logger.warning("Lost credential refresh lease: %s", name, exc_info=True)
+ return
+
+ heartbeat = Thread(target=renew, daemon=True, name=f"CredentialRefresh({credential_id})")
+ try:
+ heartbeat.start()
+ yield lease
+ finally:
+ stopped.set()
+ if heartbeat.ident is not None:
+ heartbeat.join(timeout=2)
+ try:
+ lock.release()
+ except (LockError, RedisError):
+ logger.warning("Failed to release credential refresh lease: %s", name, exc_info=True)
diff --git a/api/services/credentials/refresh.py b/api/services/credentials/refresh.py
new file mode 100644
index 00000000000..477b27baecc
--- /dev/null
+++ b/api/services/credentials/refresh.py
@@ -0,0 +1,35 @@
+"""Coordination contracts for refreshing one persisted provider credential."""
+
+from contextlib import AbstractContextManager
+from typing import Literal, Protocol
+
+
+def credential_needs_refresh(expires_at: int, *, now: float) -> bool:
+ return expires_at != -1 and expires_at - 60 < int(now)
+
+
+class CredentialRefreshUnavailableError(RuntimeError):
+ """Temporary coordination failure; preserve it until the caller chooses a retry response."""
+
+ def __init__(self) -> None:
+ super().__init__("Credential refresh is temporarily unavailable. Please retry.")
+
+
+class CredentialRefreshLease(Protocol):
+ def ensure_owned(self) -> None:
+ """Renew ownership or fail before issuing a refresh or persisting its result."""
+ ...
+
+
+class CredentialRefreshLock(Protocol):
+ def acquire(
+ self, *, credential_type: Literal["tool", "datasource"], workspace_id: str, credential_id: str
+ ) -> AbstractContextManager[CredentialRefreshLease]:
+ """Serialize refreshes across workers, with bounded waiting and a renewed lease.
+
+ Callers reload the credential after acquisition and keep this lease through
+ the external refresh and the committed conditional write. No database
+ transaction may span the external request. Failure never permits an
+ unlocked refresh or a return of the original expired credential.
+ """
+ ...
diff --git a/api/services/credentials/tool_runtime.py b/api/services/credentials/tool_runtime.py
new file mode 100644
index 00000000000..15fb19cf99d
--- /dev/null
+++ b/api/services/credentials/tool_runtime.py
@@ -0,0 +1,122 @@
+"""Resolve tool credentials through scoped snapshots and serialized OAuth refreshes."""
+
+import time
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Protocol
+
+from services.credentials.refresh import CredentialRefreshLock, credential_needs_refresh
+
+
+@dataclass(frozen=True, slots=True)
+class ToolRuntimeCredential:
+ """Detached provider state; absent ciphertext represents empty credentials."""
+
+ id: str
+ workspace_id: str
+ owner_id: str
+ provider: str
+ credential_type: str
+ encrypted_credentials: str | None
+ expires_at: int
+
+
+@dataclass(frozen=True, slots=True)
+class RefreshedToolCredential:
+ credentials: Mapping[str, object]
+ expires_at: int
+
+
+@dataclass(frozen=True, slots=True)
+class ResolvedToolCredential:
+ credentials: Mapping[str, object]
+ credential_type: str
+
+
+class ToolCredentialNotFoundError(ValueError):
+ pass
+
+
+class ToolCredentialRefreshError(ValueError):
+ pass
+
+
+class ToolCredentialStore(Protocol):
+ def get_tool_credential(
+ self, *, workspace_id: str, providers: tuple[str, ...], credential_id: str | None
+ ) -> ToolRuntimeCredential | None:
+ """Return a detached snapshot, scoped to workspace and provider, preferring the default when ID is absent."""
+ ...
+
+ def update_tool_credential_if_unchanged(
+ self, *, record: ToolRuntimeCredential, encrypted_credentials: str, expires_at: int
+ ) -> bool:
+ """Commit in a short transaction, guarding the owner, auth type, expiry and previous ciphertext."""
+ ...
+
+
+class ToolCredentialRuntime(Protocol):
+ def check_policy(self, record: ToolRuntimeCredential) -> None: ...
+
+ def decrypt(self, record: ToolRuntimeCredential) -> dict[str, object]: ...
+
+ def encrypt(self, record: ToolRuntimeCredential, credentials: Mapping[str, object]) -> str: ...
+
+ def refresh(self, record: ToolRuntimeCredential, credentials: Mapping[str, object]) -> RefreshedToolCredential: ...
+
+ def invalidate(self, record: ToolRuntimeCredential) -> None: ...
+
+
+class ToolCredentialService:
+ def __init__(
+ self,
+ *,
+ credentials: ToolCredentialStore,
+ runtime: ToolCredentialRuntime,
+ refresh_lock: CredentialRefreshLock,
+ providers: tuple[str, ...],
+ ) -> None:
+ self._credentials = credentials
+ self._runtime = runtime
+ self._refresh_lock = refresh_lock
+ self._providers = providers
+
+ def resolve(self, *, workspace_id: str, credential_id: str | None) -> ResolvedToolCredential:
+ record = self._load(workspace_id=workspace_id, credential_id=credential_id)
+ self._runtime.check_policy(record)
+ if not credential_needs_refresh(record.expires_at, now=time.time()):
+ return self._resolved(record)
+
+ with self._refresh_lock.acquire(
+ credential_type="tool", workspace_id=workspace_id, credential_id=record.id
+ ) as lease:
+ record = self._load(workspace_id=workspace_id, credential_id=record.id)
+ self._runtime.check_policy(record)
+ if not credential_needs_refresh(record.expires_at, now=time.time()):
+ return self._resolved(record)
+ decrypted = self._runtime.decrypt(record)
+ lease.ensure_owned()
+ refreshed = self._runtime.refresh(record, decrypted)
+ encrypted = self._runtime.encrypt(record, refreshed.credentials)
+ lease.ensure_owned()
+ if not self._credentials.update_tool_credential_if_unchanged(
+ record=record, encrypted_credentials=encrypted, expires_at=refreshed.expires_at
+ ):
+ latest = self._load(workspace_id=workspace_id, credential_id=record.id)
+ self._runtime.check_policy(latest)
+ if credential_needs_refresh(latest.expires_at, now=time.time()):
+ raise ToolCredentialRefreshError("Credential changed while it was being refreshed. Please retry.")
+ return self._resolved(latest)
+ self._runtime.invalidate(record)
+ return ResolvedToolCredential(refreshed.credentials, record.credential_type)
+
+ def _load(self, *, workspace_id: str, credential_id: str | None) -> ToolRuntimeCredential:
+ record = self._credentials.get_tool_credential(
+ workspace_id=workspace_id, providers=self._providers, credential_id=credential_id
+ )
+ if record is None:
+ raise ToolCredentialNotFoundError("Tool credential not found")
+ return record
+
+ def _resolved(self, record: ToolRuntimeCredential) -> ResolvedToolCredential:
+ return ResolvedToolCredential(self._runtime.decrypt(record), record.credential_type)
diff --git a/api/services/credentials/tool_runtime_adapters.py b/api/services/credentials/tool_runtime_adapters.py
new file mode 100644
index 00000000000..b452f1e95b4
--- /dev/null
+++ b/api/services/credentials/tool_runtime_adapters.py
@@ -0,0 +1,126 @@
+"""Provider policy, encryption and OAuth adapters for runtime tool credentials."""
+
+import json
+import logging
+from collections.abc import Mapping
+from hashlib import sha256
+
+from core.entities import PluginCredentialType
+from core.helper.credential_utils import runtime_check_credential_policy_compliance
+from core.helper.provider_cache import ToolProviderCredentialsCache
+from core.plugin.entities.plugin_daemon import CredentialType
+from core.plugin.impl.oauth import OAuthHandler
+from core.tools.builtin_tool.provider import BuiltinToolProviderController
+from core.tools.errors import ToolProviderCredentialValidationError
+from core.tools.plugin_tool.provider import PluginToolProviderController
+from core.tools.utils.encryption import ProviderConfigEncrypter, create_provider_encrypter
+from models.provider_ids import ToolProviderID
+from services.credentials.tool_runtime import (
+ RefreshedToolCredential,
+ ToolCredentialNotFoundError,
+ ToolCredentialRefreshError,
+ ToolCredentialService,
+ ToolRuntimeCredential,
+)
+from services.tools.builtin_tools_manage_service import BuiltinToolManageService
+
+logger = logging.getLogger(__name__)
+
+
+class ProviderToolCredentialRuntime:
+ def __init__(self, *, controller: BuiltinToolProviderController, provider_id: str, console_api_url: str) -> None:
+ self._controller = controller
+ self._provider_id = provider_id
+ self._console_api_url = console_api_url
+
+ def resolve(
+ self, *, service: ToolCredentialService, workspace_id: str, credential_id: str | None
+ ) -> tuple[Mapping[str, object], CredentialType]:
+ """Map credential failures to tool errors, retaining temporary coordination failures."""
+ try:
+ resolved = service.resolve(workspace_id=workspace_id, credential_id=credential_id)
+ except ToolCredentialNotFoundError as exc:
+ if credential_id is not None:
+ message = f"Tool credential {credential_id} has been deleted. Select or authorize another credential."
+ elif isinstance(self._controller, PluginToolProviderController):
+ message = (
+ f"No workspace credential is configured for tool provider {self._provider_id}. "
+ "Authorize the provider or select a credential."
+ )
+ else:
+ message = (
+ f"No credential is configured for built-in tool provider {self._provider_id}. "
+ "Authorize the provider or select a credential."
+ )
+ raise ToolProviderCredentialValidationError(message) from exc
+ except ToolCredentialRefreshError as exc:
+ raise ToolProviderCredentialValidationError(str(exc)) from exc
+ return resolved.credentials, CredentialType(resolved.credential_type)
+
+ def check_policy(self, record: ToolRuntimeCredential) -> None:
+ runtime_check_credential_policy_compliance(
+ credential_id=record.id,
+ provider=self._provider_id,
+ credential_type=PluginCredentialType.TOOL,
+ check_existence=False,
+ )
+
+ def _cache(self, record: ToolRuntimeCredential) -> ToolProviderCredentialsCache:
+ return ToolProviderCredentialsCache(
+ tenant_id=record.workspace_id,
+ provider=self._provider_id,
+ credential_id=record.id,
+ fingerprint=(
+ sha256((record.encrypted_credentials or "{}").encode()).hexdigest()
+ if record.credential_type == "oauth2"
+ else None
+ ),
+ )
+
+ def _encrypter(self, record: ToolRuntimeCredential) -> ProviderConfigEncrypter:
+ encrypter, _ = create_provider_encrypter(
+ tenant_id=record.workspace_id,
+ config=[
+ item.to_basic_provider_config()
+ for item in self._controller.get_credentials_schema_by_type(CredentialType(record.credential_type))
+ ],
+ cache=self._cache(record),
+ )
+ return encrypter
+
+ def decrypt(self, record: ToolRuntimeCredential) -> dict[str, object]:
+ return dict(self._encrypter(record).decrypt(json.loads(record.encrypted_credentials or "{}")))
+
+ def encrypt(self, record: ToolRuntimeCredential, credentials: Mapping[str, object]) -> str:
+ return json.dumps(self._encrypter(record).encrypt(credentials))
+
+ def invalidate(self, record: ToolRuntimeCredential) -> None:
+ self._cache(record).delete()
+
+ def refresh(self, record: ToolRuntimeCredential, credentials: Mapping[str, object]) -> RefreshedToolCredential:
+ provider = ToolProviderID(self._provider_id)
+ controller = self._controller if isinstance(self._controller, PluginToolProviderController) else None
+ client = BuiltinToolManageService.get_oauth_client(
+ record.workspace_id, self._provider_id, provider_controller=controller
+ )
+ identifier = (
+ client.plugin_unique_identifier if client else (controller.plugin_unique_identifier if controller else None)
+ )
+ try:
+ refreshed = OAuthHandler().refresh_credentials(
+ tenant_id=record.workspace_id,
+ user_id=record.owner_id,
+ plugin_id=provider.plugin_id,
+ provider=provider.provider_name,
+ redirect_uri=f"{self._console_api_url}/console/api/oauth/plugin/{self._provider_id}/tool/callback",
+ system_credentials=dict(client.credentials) if client is not None else {},
+ plugin_unique_identifier=identifier,
+ credentials=credentials,
+ )
+ except Exception as exc:
+ logger.warning("Failed to refresh OAuth credentials for tool provider %s", self._provider_id, exc_info=True)
+ raise ToolCredentialRefreshError(
+ f"OAuth credential for tool provider {self._provider_id} could not be refreshed. "
+ "Reauthorize or select another credential."
+ ) from exc
+ return RefreshedToolCredential(refreshed.credentials, refreshed.expires_at)
diff --git a/api/services/data_source/credential_gateway.py b/api/services/data_source/credential_gateway.py
index 75dc5cf7cf4..8f4e1501140 100644
--- a/api/services/data_source/credential_gateway.py
+++ b/api/services/data_source/credential_gateway.py
@@ -6,6 +6,11 @@ from collections.abc import Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import Protocol
+from services.credentials.refresh import (
+ CredentialRefreshLock,
+ CredentialRefreshUnavailableError,
+ credential_needs_refresh,
+)
from services.data_source.entities.credential import DatasourceCredentialRecord, DatasourceOAuthClientConfigRecord
logger = logging.getLogger(__name__)
@@ -27,6 +32,7 @@ def resolve_stored_notion_access_token(
default must not change the integration used to read an existing document.
Preserve indexing's environment fallback when credential resolution fails,
including refresh failures. Actor-selected imports do not use this policy.
+ Without a fallback, preserve temporary refresh failures so callers can retry.
"""
credential_error: Exception | None = None
if credential_id:
@@ -38,6 +44,8 @@ def resolve_stored_notion_access_token(
credential_error = error
if not integration_token:
+ if isinstance(credential_error, CredentialRefreshUnavailableError):
+ raise credential_error
raise NotionCredentialUnavailableError(
"Must specify `integration_token` or set environment variable `NOTION_INTEGRATION_TOKEN`."
) from credential_error
@@ -63,7 +71,9 @@ class DatasourceProviderCredentialStore(Protocol):
def update_if_unchanged(
self, *, record: DatasourceCredentialRecord, encrypted_credentials: Mapping[str, object], expires_at: int
- ) -> bool: ...
+ ) -> bool:
+ """Commit a scoped write guarded by owner, auth type, ciphertext and expiry; return false on a conflict."""
+ ...
class DatasourceCredentialStore(Protocol):
@@ -189,11 +199,13 @@ class ActorAwareDatasourceCredentialGateway:
credentials: DatasourceCredentialStore,
codec: DatasourceCredentialCodec,
refresher: DatasourceCredentialRefresher,
+ refresh_lock: CredentialRefreshLock,
now: Callable[[], float] = time.time,
) -> None:
self._credentials = credentials
self._codec = codec
self._refresher = refresher
+ self._refresh_lock = refresh_lock
self._now = now
def resolve(
@@ -224,6 +236,7 @@ class ActorAwareDatasourceCredentialGateway:
refresher=self._refresher,
update=self._credentials.update_if_unchanged,
load_latest=load_record,
+ refresh_lock=self._refresh_lock,
now=self._now,
)
@@ -237,11 +250,13 @@ class TrustedStoredDatasourceCredentialGateway:
credentials: TrustedDatasourceCredentialStore,
codec: DatasourceCredentialCodec,
refresher: DatasourceCredentialRefresher,
+ refresh_lock: CredentialRefreshLock,
now: Callable[[], float] = time.time,
) -> None:
self._credentials = credentials
self._codec = codec
self._refresher = refresher
+ self._refresh_lock = refresh_lock
self._now = now
def resolve_for_document(
@@ -270,19 +285,16 @@ class TrustedStoredDatasourceCredentialGateway:
record = load_record()
return resolve_credential_snapshot(
record,
- actor_id=record.owner_id,
+ actor_id=None,
codec=self._codec,
refresher=self._refresher,
update=self._credentials.update_if_unchanged,
load_latest=load_record,
+ refresh_lock=self._refresh_lock,
now=self._now,
)
-def credential_needs_refresh(expires_at: int, *, now: float) -> bool:
- return expires_at != -1 and expires_at - 60 < int(now)
-
-
class CredentialSnapshotUpdate(Protocol):
def __call__(
self, *, record: DatasourceCredentialRecord, encrypted_credentials: Mapping[str, object], expires_at: int
@@ -297,23 +309,38 @@ def resolve_credential_snapshot(
refresher: DatasourceCredentialRefresher,
update: CredentialSnapshotUpdate,
load_latest: Callable[[], DatasourceCredentialRecord],
+ refresh_lock: CredentialRefreshLock,
now: Callable[[], float],
) -> dict[str, object]:
- decrypted = codec.decrypt(record)
+ """Refresh under one lease; an absent actor selects the persisted credential owner.
+
+ Every caller, including legacy provider execution, uses the same credential
+ lock. Reloading also rechecks the caller's visibility or stored owner chain.
+ """
if not credential_needs_refresh(record.expires_at, now=now()):
- return decrypted
- if actor_id is None:
- raise DatasourceCredentialRefreshError(record.id)
- refreshed = refresher.refresh(
- workspace_id=record.workspace_id, actor_id=actor_id, record=record, credentials=decrypted
- )
- if update(
- record=record,
- encrypted_credentials=codec.encrypt(record, refreshed.credentials),
- expires_at=refreshed.expires_at,
- ):
- return dict(refreshed.credentials)
- latest = load_latest()
- if credential_needs_refresh(latest.expires_at, now=now()):
- raise DatasourceCredentialConcurrentUpdateError()
- return codec.decrypt(latest)
+ return codec.decrypt(record)
+ with refresh_lock.acquire(
+ credential_type="datasource", workspace_id=record.workspace_id, credential_id=record.id
+ ) as lease:
+ latest = load_latest()
+ if (latest.workspace_id, latest.id) != (record.workspace_id, record.id):
+ raise DatasourceCredentialConcurrentUpdateError()
+ record = latest
+ if not credential_needs_refresh(record.expires_at, now=now()):
+ return codec.decrypt(record)
+ refresh_actor = actor_id if actor_id is not None else record.owner_id
+ if refresh_actor is None:
+ raise DatasourceCredentialRefreshError(record.id)
+ decrypted = codec.decrypt(record)
+ lease.ensure_owned()
+ refreshed = refresher.refresh(
+ workspace_id=record.workspace_id, actor_id=refresh_actor, record=record, credentials=decrypted
+ )
+ encrypted = codec.encrypt(record, refreshed.credentials)
+ lease.ensure_owned()
+ if update(record=record, encrypted_credentials=encrypted, expires_at=refreshed.expires_at):
+ return dict(refreshed.credentials)
+ latest = load_latest()
+ if credential_needs_refresh(latest.expires_at, now=now()):
+ raise DatasourceCredentialConcurrentUpdateError()
+ return codec.decrypt(latest)
diff --git a/api/services/data_source/provider_service.py b/api/services/data_source/provider_service.py
index 3b8a73acb0d..8ca0f4fe829 100644
--- a/api/services/data_source/provider_service.py
+++ b/api/services/data_source/provider_service.py
@@ -1,10 +1,13 @@
import logging
import time
from collections.abc import Mapping
-from dataclasses import replace
from typing import TYPE_CHECKING, Any, cast
from services.credentials.query import CredentialQuery
+from services.credentials.refresh import (
+ CredentialRefreshLock,
+ credential_needs_refresh,
+)
from services.plugin.dispatch_identity import PluginOAuthClientConfig
if TYPE_CHECKING:
@@ -20,7 +23,6 @@ from core.helper.name_generator import generate_incremental_name
from core.helper.provider_cache import NoOpProviderCredentialCache
from core.plugin.entities.plugin_daemon import CredentialType, PluginDatasourceProviderEntity
from core.plugin.impl.datasource import PluginDatasourceManager
-from core.plugin.impl.oauth import OAuthHandler
from core.tools.utils.encryption import ProviderConfigCache, ProviderConfigEncrypter, create_provider_encrypter
from extensions.ext_database import db
from extensions.ext_redis import redis_client
@@ -29,14 +31,16 @@ from models.oauth import DatasourceOauthParamConfig, DatasourceOauthTenantParamC
from models.provider_ids import DatasourceProviderID
from services.data_source.credential_adapters import (
datasource_secret_variables,
- refresh_datasource_credential,
resolve_datasource_oauth_client,
transform_secret_fields,
)
from services.data_source.credential_gateway import (
+ DatasourceCredentialCodec,
DatasourceCredentialError,
+ DatasourceCredentialNotFoundError,
+ DatasourceCredentialRefresher,
DatasourceProviderCredentialStore,
- credential_needs_refresh,
+ resolve_credential_snapshot,
resolve_stored_notion_access_token,
)
from services.data_source.entities.credential import DatasourceCredentialRecord
@@ -65,9 +69,20 @@ class DatasourceProviderService:
Datasource provider credentials and authorization management.
"""
- def __init__(self, *, credentials: DatasourceProviderCredentialStore) -> None:
+ def __init__(
+ self,
+ *,
+ credentials: DatasourceProviderCredentialStore,
+ codec: DatasourceCredentialCodec,
+ refresher: DatasourceCredentialRefresher,
+ refresh_lock: CredentialRefreshLock,
+ provider_manager: PluginDatasourceManager,
+ ) -> None:
self._credentials = credentials
- self.provider_manager = PluginDatasourceManager()
+ self._codec = codec
+ self._refresher = refresher
+ self._refresh_lock = refresh_lock
+ self.provider_manager = provider_manager
def get_stored_notion_access_token(self, *, tenant_id: str, credential_id: str | None) -> str:
"""Resolve a stored Notion credential with the configured integration fallback."""
@@ -90,98 +105,6 @@ class DatasourceProviderService:
integration_token=dify_config.NOTION_INTEGRATION_TOKEN,
)
- @staticmethod
- def _should_refresh_credentials(
- datasource_provider: DatasourceProvider | DatasourceCredentialRecord, now: int | None = None
- ) -> bool:
- current_time = int(time.time()) if now is None else now
- return credential_needs_refresh(datasource_provider.expires_at, now=current_time)
-
- def _refresh_datasource_credentials(
- self,
- tenant_id: str,
- provider: str,
- plugin_id: str,
- datasource_provider: DatasourceProvider | DatasourceCredentialRecord,
- current_user: Any,
- ) -> tuple[dict[str, Any], int]:
- datasource_provider_id = DatasourceProviderID(f"{plugin_id}/{provider}")
- provider_name = datasource_provider_id.provider_name
- provider_declaration = self.provider_manager.fetch_datasource_provider(
- tenant_id=tenant_id, provider_id=str(datasource_provider_id)
- )
- plugin_unique_identifier = provider_declaration.plugin_unique_identifier
- credential_id = getattr(datasource_provider, "id", None)
- credential_name = getattr(datasource_provider, "name", None)
- logger.info(
- "Refreshing datasource credentials for provider %s",
- provider_name,
- extra={
- "tenant_id": tenant_id,
- "plugin_id": datasource_provider_id.plugin_id,
- "provider": provider_name,
- "credential_id": credential_id,
- "credential_name": credential_name,
- "expires_at": datasource_provider.expires_at,
- },
- )
- decrypted_credentials = self.decrypt_datasource_provider_credentials(
- tenant_id=tenant_id,
- datasource_provider=datasource_provider,
- plugin_id=plugin_id,
- provider=provider,
- provider_declaration=provider_declaration,
- )
- system_credentials = self.get_oauth_client(
- tenant_id, datasource_provider_id, provider_declaration=provider_declaration
- )
- try:
- refreshed_credentials = refresh_datasource_credential(
- OAuthHandler(),
- workspace_id=tenant_id,
- actor_id=current_user.id,
- provider_id=datasource_provider_id,
- system_credentials=system_credentials or PluginOAuthClientConfig(plugin_unique_identifier, {}),
- credentials=decrypted_credentials,
- )
- except Exception as exc:
- message = (
- f"Failed to refresh datasource credentials for provider {provider_name}"
- f" (credential: {credential_name or credential_id or 'unknown'})"
- )
- logger.exception(
- message,
- extra={
- "tenant_id": tenant_id,
- "plugin_id": datasource_provider_id.plugin_id,
- "provider": provider_name,
- "credential_id": credential_id,
- "credential_name": credential_name,
- },
- )
- raise ValueError(f"{message}: {exc}") from exc
- encrypted_credentials = self.encrypt_datasource_provider_credentials(
- tenant_id=tenant_id,
- raw_credentials=refreshed_credentials.credentials,
- provider=provider,
- provider_declaration=provider_declaration,
- plugin_id=plugin_id,
- datasource_provider=datasource_provider,
- )
- logger.info(
- "Refreshed datasource credentials for provider %s",
- provider_name,
- extra={
- "tenant_id": tenant_id,
- "plugin_id": datasource_provider_id.plugin_id,
- "provider": provider_name,
- "credential_id": credential_id,
- "credential_name": credential_name,
- "expires_at": refreshed_credentials.expires_at,
- },
- )
- return encrypted_credentials, refreshed_credentials.expires_at
-
def remove_oauth_custom_client_params(self, tenant_id: str, datasource_provider_id: DatasourceProviderID):
"""
remove oauth custom client params
@@ -259,36 +182,42 @@ class DatasourceProviderService:
def _resolve_provider_credentials(
self, record: DatasourceCredentialRecord, *, current_user: Any | None = None
) -> dict[str, Any]:
- """Refresh a snapshot; the store owns each read and conditional write transaction."""
- if self._should_refresh_credentials(record):
- encrypted, expires_at = self._refresh_datasource_credentials(
- tenant_id=record.workspace_id,
+ """Resolve through the same refresh policy and lease as actor and stored-document callers."""
+
+ def load_latest() -> DatasourceCredentialRecord:
+ latest = self._credentials.get_provider_credential(
+ workspace_id=record.workspace_id,
provider=record.provider,
plugin_id=record.plugin_id,
- datasource_provider=record,
- current_user=current_user if current_user is not None else get_current_user(),
+ credential_id=record.id,
)
- saved = self._credentials.update_if_unchanged(
- record=record, encrypted_credentials=encrypted, expires_at=expires_at
+ if latest is None:
+ raise DatasourceCredentialNotFoundError()
+ return latest
+
+ actor_id = current_user.id if current_user is not None else None
+ if (
+ actor_id is None
+ and record.owner_id is None
+ and credential_needs_refresh(record.expires_at, now=time.time())
+ ):
+ actor_id = get_current_user().id
+ try:
+ return resolve_credential_snapshot(
+ record,
+ actor_id=actor_id,
+ codec=self._codec,
+ refresher=self._refresher,
+ update=self._credentials.update_if_unchanged,
+ load_latest=load_latest,
+ refresh_lock=self._refresh_lock,
+ now=time.time,
)
- if saved:
- record = replace(record, encrypted_credentials=encrypted, expires_at=expires_at)
- else:
- latest = self._credentials.get_provider_credential(
- workspace_id=record.workspace_id,
- provider=record.provider,
- plugin_id=record.plugin_id,
- credential_id=record.id,
- )
- if latest is None or self._should_refresh_credentials(latest):
- raise ValueError("Credential changed while it was being refreshed")
- record = latest
- return self.decrypt_datasource_provider_credentials(
- tenant_id=record.workspace_id,
- datasource_provider=record,
- plugin_id=record.plugin_id,
- provider=record.provider,
- )
+ except DatasourceCredentialError as error:
+ raise ValueError(
+ f"Failed to refresh datasource credentials for provider {record.provider}"
+ f" (credential: {record.name or record.id}): {error}"
+ ) from error
def get_all_datasource_credentials_by_provider(
self,
diff --git a/api/services/knowledge/indexing/adapters/estimate.py b/api/services/knowledge/indexing/adapters/estimate.py
index d6279f2d2ea..08233aab83c 100644
--- a/api/services/knowledge/indexing/adapters/estimate.py
+++ b/api/services/knowledge/indexing/adapters/estimate.py
@@ -26,6 +26,7 @@ from graphon.model_runtime.entities.model_entities import ModelType
from models.dataset import Dataset, DatasetProcessRule
from repositories.knowledge.dataset_repository import _get_dataset
from repositories.knowledge.upload_file_repository import SQLAlchemyKnowledgeUploadRepository
+from services.credentials.refresh import CredentialRefreshUnavailableError
from services.knowledge.indexing.estimate import (
IndexingEstimateExecutionError,
IndexingEstimateProviderUnavailableError,
@@ -87,6 +88,8 @@ class IndexingEstimateAdapter:
)
except (LLMBadRequestError, ProviderTokenNotInitError, PluginDaemonClientSideError) as error:
raise IndexingEstimateProviderUnavailableError(error.description) from error
+ except CredentialRefreshUnavailableError:
+ raise
except Exception as error:
raise IndexingEstimateExecutionError(str(error)) from error
diff --git a/api/services/knowledge/indexing/estimate.py b/api/services/knowledge/indexing/estimate.py
index 6e75705e27c..d9c1b5d98f4 100644
--- a/api/services/knowledge/indexing/estimate.py
+++ b/api/services/knowledge/indexing/estimate.py
@@ -6,6 +6,7 @@ from typing import NamedTuple, Never, Protocol
from core.entities.knowledge_entities import IndexingEstimate
from core.rag.extractor.entity.extract_setting import ExtractSetting
from machinery.context import RequestContext
+from services.credentials.refresh import CredentialRefreshUnavailableError
from services.data_source.credential_gateway import DatasourceCredentialError
from services.data_source.entities.notion_import import NotionPageType
from services.knowledge.dataset_access import DatasetAccess, DatasetNotFoundError
@@ -328,7 +329,7 @@ class IndexingEstimateApplicationService:
@staticmethod
def _translate_error(error: Exception) -> Never:
- if isinstance(error, IndexingEstimateApplicationError):
+ if isinstance(error, (IndexingEstimateApplicationError, CredentialRefreshUnavailableError)):
raise error
if isinstance(error, UnsupportedStoredSourceError):
raise UnsupportedEstimateSourceError(error.source_type) from error
diff --git a/api/tests/helpers/unit/credential_refresh.py b/api/tests/helpers/unit/credential_refresh.py
new file mode 100644
index 00000000000..ed98734157b
--- /dev/null
+++ b/api/tests/helpers/unit/credential_refresh.py
@@ -0,0 +1,72 @@
+"""Concrete credential ports for unit tests, including actual thread exclusion."""
+
+from __future__ import annotations
+
+from collections.abc import Generator
+from contextlib import contextmanager
+from threading import Condition, Lock
+from typing import TYPE_CHECKING, Literal
+
+from services.credentials.refresh import CredentialRefreshLease, CredentialRefreshUnavailableError
+from services.data_source.credential_gateway import DatasourceProviderCredentialStore
+
+if TYPE_CHECKING:
+ from services.data_source.provider_service import DatasourceProviderService
+
+
+class ThreadCredentialRefreshLock:
+ def __init__(self, *, wait_seconds: float = 2) -> None:
+ self.wait_seconds = wait_seconds
+ self.keys: list[tuple[str, str, str]] = []
+ self.lost = False
+ self.unavailable = False
+ self._condition = Condition()
+ self._locks: dict[tuple[str, str, str], Lock] = {}
+
+ @contextmanager
+ def acquire(
+ self, *, credential_type: Literal["tool", "datasource"], workspace_id: str, credential_id: str
+ ) -> Generator[CredentialRefreshLease]:
+ key = (credential_type, workspace_id, credential_id)
+ with self._condition:
+ self.keys.append(key)
+ lock = self._locks.setdefault(key, Lock())
+ self._condition.notify_all()
+ if self.unavailable or not lock.acquire(timeout=self.wait_seconds):
+ raise CredentialRefreshUnavailableError()
+ try:
+ yield self
+ finally:
+ lock.release()
+
+ def ensure_owned(self) -> None:
+ if self.lost:
+ raise CredentialRefreshUnavailableError()
+
+ def wait_for_callers(self, count: int) -> None:
+ """Keep the provider request open until competing callers have attempted the lock."""
+ with self._condition:
+ assert self._condition.wait_for(lambda: len(self.keys) >= count, timeout=self.wait_seconds)
+
+
+def build_datasource_provider_service(*, credentials: DatasourceProviderCredentialStore) -> DatasourceProviderService:
+ from core.plugin.impl.datasource import PluginDatasourceManager
+ from core.plugin.impl.oauth import OAuthHandler
+ from services.data_source.credential_adapters import (
+ OAuthDatasourceCredentialRefresher,
+ PluginDatasourceCredentialCodec,
+ PluginDatasourceOAuthClientResolver,
+ )
+ from services.data_source.provider_service import DatasourceProviderService
+
+ manager = PluginDatasourceManager()
+ return DatasourceProviderService(
+ credentials=credentials,
+ codec=PluginDatasourceCredentialCodec(provider_manager=manager),
+ refresher=OAuthDatasourceCredentialRefresher(
+ oauth_clients=PluginDatasourceOAuthClientResolver(configs=credentials, provider_manager=manager),
+ oauth_handler=OAuthHandler(),
+ ),
+ refresh_lock=ThreadCredentialRefreshLock(),
+ provider_manager=manager,
+ )
diff --git a/api/tests/unit_tests/controllers/common/test_errors.py b/api/tests/unit_tests/controllers/common/test_errors.py
index 417882ceef0..90fb8f3e35d 100644
--- a/api/tests/unit_tests/controllers/common/test_errors.py
+++ b/api/tests/unit_tests/controllers/common/test_errors.py
@@ -20,9 +20,11 @@ from controllers.common.errors import (
UnsupportedFileTypeError,
)
from controllers.console import api as console_api
+from controllers.inner_api import api as inner_api
from controllers.service_api import api as service_api
from libs.exception import BaseHTTPException
from libs.external_api import ExternalApi
+from services.credentials.refresh import CredentialRefreshUnavailableError
from services.errors.base import NoPermissionError
from services.errors.workspace import WorkspaceApplicationError
@@ -70,6 +72,25 @@ def test_permission_error_mapping_is_not_registered_on_generic_apis() -> None:
assert response.json == {"code": "unknown", "message": "Internal Server Error", "status": 500}
[email protected]("source_api", [console_api, inner_api], ids=["console", "inner_api"])
+def test_refresh_unavailability_has_one_documented_response(source_api: ExternalApi) -> None:
+ error = CredentialRefreshUnavailableError()
+ app = Flask(__name__)
+ api = Api(app, doc=False)
+ api.error_handlers = source_api.error_handlers.copy()
+
+ class Endpoint(Resource):
+ def get(self) -> ResponseReturnValue:
+ raise error
+
+ api.add_resource(Endpoint, "/error")
+ response = app.test_client().get("/error")
+
+ assert response.status_code == 503
+ assert response.json == {"code": "service_unavailable", "message": str(error), "status": 503}
+ assert source_api.resolve_error(CredentialRefreshUnavailableError) == (503, "service_unavailable")
+
+
class TestFilenameNotExistsError:
def test_defaults(self):
error = FilenameNotExistsError()
diff --git a/api/tests/unit_tests/controllers/console/datasets/test_data_source.py b/api/tests/unit_tests/controllers/console/datasets/test_data_source.py
index e887b5f8fa7..2f62faf8b7d 100644
--- a/api/tests/unit_tests/controllers/console/datasets/test_data_source.py
+++ b/api/tests/unit_tests/controllers/console/datasets/test_data_source.py
@@ -11,10 +11,10 @@ from typing import TYPE_CHECKING, cast
import pytest
from flask import Flask
from flask.testing import FlaskClient
-from flask_restx import Api
from controllers.common.errors import NotFoundError
from controllers.common.rbac import DatasetId
+from controllers.console import api as console_api
from controllers.console import console_ns
from controllers.console.app.error import ProviderNotInitializeError
from controllers.console.datasets import data_source as data_source_controller
@@ -22,8 +22,10 @@ from controllers.console.datasets.error import IndexingEstimateError
from core.entities.knowledge_entities import IndexingEstimate
from core.rbac import RBACPermission
from enums import DeploymentEdition
+from libs.external_api import ExternalApi
from libs.login import AccountWithTenant
from machinery.context import RequestContext
+from services.credentials.refresh import CredentialRefreshUnavailableError
from services.data_source.binding_application_service import (
BindingMutationResult,
DataSourceBindingApplicationService,
@@ -234,6 +236,7 @@ class RecordingNotionSource:
workspaces: tuple[NotionWorkspace, ...]
list_calls: list[tuple[str, str, str]] = field(default_factory=list)
preview_calls: list[tuple[str, str, str, str, str]] = field(default_factory=list)
+ error: Exception | None = None
def list_authorized_pages(
self,
@@ -243,6 +246,8 @@ class RecordingNotionSource:
credential_id: str,
) -> tuple[NotionWorkspace, ...]:
self.list_calls.append((workspace_id, actor_id, credential_id))
+ if self.error is not None:
+ raise self.error
return self.workspaces
def preview_page(
@@ -255,6 +260,8 @@ class RecordingNotionSource:
page_type: str,
) -> str:
self.preview_calls.append((workspace_id, actor_id, credential_id, page_id, page_type))
+ if self.error is not None:
+ raise self.error
return "preview text"
@@ -328,7 +335,9 @@ class TestDataSourceCases:
)
app = Flask(__name__)
app.config.update(TESTING=False, PROPAGATE_EXCEPTIONS=False)
- Api(app).add_namespace(console_ns, path="/console/api")
+ api = ExternalApi(app)
+ api.error_handlers = console_api.error_handlers.copy()
+ api.add_namespace(console_ns, path="/console/api")
account = AdmittedAccount(id=ACCOUNT_ID, role="owner", status="active")
account_with_tenant = AccountWithTenant(account=cast("Account", account), tenant_id=TENANT_ID)
@@ -581,6 +590,21 @@ class TestDataSourceCases:
(TENANT_ID, ACCOUNT_ID, "credential-1", PAGE_ID, page_type)
]
+ @staticmethod
+ @pytest.mark.parametrize("path", ["/notion/pre-import/pages", f"/notion/pages/{PAGE_ID}/page/preview"])
+ def test_notion_refresh_unavailable_returns_retryable_response(
+ collaborators: ControllerCollaborators, path: str
+ ) -> None:
+ error = CredentialRefreshUnavailableError()
+ collaborators.notion_source.error = error
+ response = collaborators.client.get(f"/console/api{path}", query_string={"credential_id": "credential-1"})
+ assert response.status_code == 503
+ assert response.get_json() == {
+ "code": "service_unavailable",
+ "message": str(error),
+ "status": 503,
+ }
+
@staticmethod
def test_notion_estimate_maps_nested_payload_to_shared_command(collaborators: ControllerCollaborators) -> None:
response = collaborators.client.post(
diff --git a/api/tests/unit_tests/controllers/inner_api/agent/test_tools.py b/api/tests/unit_tests/controllers/inner_api/agent/test_tools.py
index 3723ecf8a10..ce082dcfc0b 100644
--- a/api/tests/unit_tests/controllers/inner_api/agent/test_tools.py
+++ b/api/tests/unit_tests/controllers/inner_api/agent/test_tools.py
@@ -2,11 +2,26 @@
from collections.abc import Generator
from contextlib import contextmanager
-from unittest.mock import patch
+from unittest.mock import MagicMock, patch
+import pytest
from flask import Flask
+from redis.exceptions import LockNotOwnedError, RedisError
+from redis.lock import Lock
+from sqlalchemy.orm import Session
from controllers.inner_api import bp as inner_api_bp
+from core.plugin.entities.plugin_daemon import CredentialType
+from core.tools.entities.tool_entities import ToolProviderEntityWithPlugin
+from core.tools.plugin_tool.provider import PluginToolProviderController
+from core.tools.tool_manager import ToolManager
+from extensions.application_services.tools import resolve_tool_credentials
+from extensions.ext_redis import RedisClientWrapper
+from models.model import App, AppMode
+from models.tools import BuiltinToolProvider
+from services.credentials.refresh import CredentialRefreshUnavailableError
+from services.credentials.tool_runtime import ToolCredentialRefreshError
+from services.credentials.tool_runtime_adapters import ProviderToolCredentialRuntime
from services.entities.agent_tool_inner import AgentToolInvokeResponse
from services.errors.agent_tool_inner import AgentToolInnerServiceError
from tests.helpers.unit.config_override import config_overrides_context
@@ -19,7 +34,7 @@ def _headers(api_key: str | None = "inner-key") -> dict[str, str]:
return headers
-def _payload() -> dict[str, object]:
+def _payload(*, provider_type: str = "plugin", credential_id: str | None = "credential-1") -> dict[str, object]:
return {
"caller": {
"tenant_id": "tenant-1",
@@ -36,10 +51,10 @@ def _payload() -> dict[str, object]:
"agent_config_version_id": "snapshot-1",
},
"tool": {
- "provider_type": "plugin",
+ "provider_type": provider_type,
"provider_id": "langgenius/search/search",
"tool_name": "search",
- "credential_id": "credential-1",
+ "credential_id": credential_id,
"tool_parameters": {"query": "dify"},
"runtime_parameters": {"region": "us"},
},
@@ -122,3 +137,115 @@ def test_post_preserves_service_error_status_code_and_description() -> None:
body = response.get_json()
assert body["code"] == "app_tenant_mismatch"
assert body["message"] == "App does not belong to the caller tenant."
+
+
[email protected]("credential_id", [None, "1a1db43c-e9d8-4ef2-af8f-38b25424a4ad"])
[email protected]("failure", ["busy", "redis", "lost", "rejected"])
+def test_post_preserves_refresh_error_classification_through_real_credential_resolution(
+ sqlite_session: Session,
+ redis_transport: tuple[RedisClientWrapper, MagicMock],
+ monkeypatch: pytest.MonkeyPatch,
+ credential_id: str | None,
+ failure: str,
+) -> None:
+ provider_id = "langgenius/search/search"
+ credential = BuiltinToolProvider(
+ tenant_id="tenant-1",
+ user_id="user-1",
+ provider=provider_id,
+ name="Search",
+ credential_type=CredentialType.OAUTH2,
+ encrypted_credentials="{}",
+ expires_at=1,
+ is_default=True,
+ )
+ credential.id = "1a1db43c-e9d8-4ef2-af8f-38b25424a4ad"
+ sqlite_session.add_all(
+ [
+ App(
+ id="app-1",
+ tenant_id="tenant-1",
+ name="Agent",
+ mode=AppMode.AGENT_CHAT,
+ enable_site=False,
+ enable_api=False,
+ max_active_requests=None,
+ ),
+ credential,
+ ]
+ )
+ sqlite_session.commit()
+ controller = PluginToolProviderController(
+ entity=ToolProviderEntityWithPlugin.model_validate(
+ {
+ "identity": {
+ "author": "langgenius",
+ "name": "search",
+ "description": {"en_US": "Search"},
+ "icon": "search.svg",
+ "label": {"en_US": "Search"},
+ },
+ "oauth_schema": {
+ "client_schema": [],
+ "credentials_schema": [
+ {"name": "access_token", "type": "secret-input", "label": {"en_US": "Access token"}}
+ ],
+ },
+ "tools": [
+ {
+ "identity": {
+ "author": "langgenius",
+ "name": "search",
+ "provider": "search",
+ "label": {"en_US": "Search"},
+ },
+ "description": {"human": {"en_US": "Search"}, "llm": "Search"},
+ "parameters": [],
+ }
+ ],
+ }
+ ),
+ plugin_id="langgenius/search",
+ plugin_unique_identifier="langgenius/search:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ tenant_id="tenant-1",
+ )
+ redis, _ = redis_transport
+ monkeypatch.setattr("extensions.application_services.tools.redis_client", redis)
+ monkeypatch.setattr("core.helper.provider_cache.redis_client", redis)
+ monkeypatch.setattr(ToolManager, "_credential_resolver", resolve_tool_credentials)
+ app = Flask(__name__)
+ app.config["TESTING"] = True
+ app.register_blueprint(inner_api_bp)
+ rejected = ToolCredentialRefreshError("OAuth credential was rejected. Reauthorize the provider.")
+
+ with (
+ _agent_inner_auth(),
+ patch.object(ToolManager, "get_builtin_provider", return_value=controller),
+ patch("core.helper.credential_utils.check_credential_policy_compliance"),
+ patch.object(
+ Lock,
+ "acquire",
+ return_value=failure != "busy",
+ side_effect=RedisError("unavailable") if failure == "redis" else None,
+ ),
+ patch.object(Lock, "reacquire", side_effect=LockNotOwnedError("expired") if failure == "lost" else None),
+ patch.object(Lock, "release"),
+ patch.object(ProviderToolCredentialRuntime, "refresh", side_effect=rejected) as refresh,
+ ):
+ response = app.test_client().post(
+ "/inner/api/agent/tools/invoke",
+ json=_payload(provider_type="builtin", credential_id=credential_id),
+ headers=_headers(),
+ )
+
+ expected_status = 422 if failure == "rejected" else 503
+ assert response.status_code == expected_status
+ assert response.get_json() == {
+ "code": "agent_tool_credential_invalid" if failure == "rejected" else "service_unavailable",
+ "message": str(rejected) if failure == "rejected" else str(CredentialRefreshUnavailableError()),
+ "status": expected_status,
+ }
+ assert refresh.call_count == (1 if failure == "rejected" else 0)
+ sqlite_session.refresh(credential)
+ assert credential.encrypted_credentials == "{}"
+ assert credential.expires_at == 1
diff --git a/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py b/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py
index 0112481074c..4d78dd4e98a 100644
--- a/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py
+++ b/api/tests/unit_tests/controllers/service_api/dataset/rag_pipeline/test_rag_pipeline_workflow.py
@@ -43,7 +43,6 @@ from models.enums import CreatorUserRole
from models.model import UploadFile
from repositories.credentials.query_repository import CredentialQueryRepository
from repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository
-from services.data_source.provider_service import DatasourceProviderService
from services.errors.file import FileTooLargeError as FileTooLargeServiceError
from services.errors.file import UnsupportedFileTypeError
from services.rag_pipeline.entity.pipeline_service_api_entities import (
@@ -51,6 +50,7 @@ from services.rag_pipeline.entity.pipeline_service_api_entities import (
PipelineRunApiEntity,
)
from services.rag_pipeline.rag_pipeline import RagPipelineService
+from tests.helpers.unit.credential_refresh import build_datasource_provider_service
from tests.helpers.unit.model_factories import make_upload_file
"""
@@ -101,7 +101,7 @@ class TestRagPipelineWorkflowCases:
session_proxy = scoped_session(sqlite_session_factory)
monkeypatch.setattr(workflow_module.db, "session", session_proxy)
query = CredentialQueryRepository(session_factory=sqlite_session_factory)
- providers = DatasourceProviderService(
+ providers = build_datasource_provider_service(
credentials=SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)
)
monkeypatch.setattr(
diff --git a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py
index ec3a12bf1bb..925eb1fb7fd 100644
--- a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py
+++ b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py
@@ -21,8 +21,8 @@ from models.model import EndUser
from models.workflow import Workflow, WorkflowType
from repositories.knowledge.document_repository import SQLAlchemyDocumentRepository
from services.data_source.credential_gateway import DatasourceProviderCredentialStore
-from services.data_source.provider_service import DatasourceProviderService
from services.workflow_run_agg import WorkflowRunAgg
+from tests.helpers.unit.credential_refresh import build_datasource_provider_service
TENANT_ID = "00000000-0000-0000-0000-000000000001"
PIPELINE_ID = "00000000-0000-0000-0000-000000000002"
@@ -47,7 +47,7 @@ def generator(mocker: MockerFixture, sqlite_engine: Engine):
gen = module.PipelineGenerator(
execution_driver=WorkflowRunAgg.run,
documents=SQLAlchemyDocumentRepository(session_factory=sessionmaker(bind=sqlite_engine)),
- datasource_providers=DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
diff --git a/api/tests/unit_tests/core/datasource/website_crawl/test_website_crawl_plugin.py b/api/tests/unit_tests/core/datasource/website_crawl/test_website_crawl_plugin.py
index acf2f4a8446..24b80342fe8 100644
--- a/api/tests/unit_tests/core/datasource/website_crawl/test_website_crawl_plugin.py
+++ b/api/tests/unit_tests/core/datasource/website_crawl/test_website_crawl_plugin.py
@@ -1,11 +1,10 @@
from collections.abc import Generator
-from unittest.mock import Mock, create_autospec, patch
+from unittest.mock import Mock, patch
import httpx
import pytest
from pytest_mock import MockerFixture
-import services.data_source.website_service as website_service_module
from core.datasource.__base.datasource_runtime import DatasourceRuntime
from core.datasource.entities.datasource_entities import (
DatasourceEntity,
@@ -18,7 +17,6 @@ from core.datasource.entities.datasource_entities import (
from core.datasource.website_crawl.website_crawl_plugin import WebsiteCrawlDatasourcePlugin
from core.datasource.website_crawl.website_crawl_provider import WebsiteCrawlDatasourcePluginProviderController
from core.rag.extractor.watercrawl.provider import WaterCrawlProvider
-from services.data_source.credential_gateway import DatasourceProviderCredentialStore
from services.data_source.website_service import CrawlOptions, CrawlRequest, WebsiteService
from tests.helpers.unit.core.datasource.factories import datasource_entity
@@ -516,11 +514,7 @@ class TestWebsiteCrawlCases:
options={"limit": 10, "crawl_sub_pages": True, "only_main_content": True},
)
- result = WebsiteService(
- providers=website_service_module.DatasourceProviderService(
- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
- )
- ).crawl_url(api_request)
+ result = WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)
assert result["status"] == "active"
assert result["job_id"] == "job-123"
@@ -550,11 +544,7 @@ class TestWebsiteCrawlCases:
options={"limit": 20, "crawl_sub_pages": True, "max_depth": 2},
)
- result = WebsiteService(
- providers=website_service_module.DatasourceProviderService(
- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
- )
- ).crawl_url(api_request)
+ result = WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)
assert result["status"] == "active"
assert result["job_id"] == "job-456"
@@ -585,11 +575,7 @@ class TestWebsiteCrawlCases:
options={"limit": 15, "crawl_sub_pages": True, "use_sitemap": True},
)
- result = WebsiteService(
- providers=website_service_module.DatasourceProviderService(
- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
- )
- ).crawl_url(api_request)
+ result = WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)
assert result["status"] == "active"
assert result["job_id"] == "task-789"
@@ -880,11 +866,7 @@ class TestWebsiteCrawlCases:
# The error should be raised when trying to crawl with invalid provider
with pytest.raises(ValueError, match="Invalid provider"):
- WebsiteService(
- providers=website_service_module.DatasourceProviderService(
- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
- )
- ).crawl_url(api_request)
+ WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)
def test_missing_api_key_error(self, mocker: MockerFixture) -> None:
"""Test that missing API key is handled properly at the httpx client level."""
@@ -1311,11 +1293,7 @@ class TestWebsiteCrawlCases:
},
)
- result = WebsiteService(
- providers=website_service_module.DatasourceProviderService(
- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
- )
- ).crawl_url(api_request)
+ result = WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)
assert result["status"] == "active"
assert result["job_id"] == "prompt-job-123"
@@ -1361,11 +1339,7 @@ class TestWebsiteCrawlCases:
provider="jinareader", url="https://example.com/page", options={"crawl_sub_pages": False, "limit": 1}
)
- result = WebsiteService(
- providers=website_service_module.DatasourceProviderService(
- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
- )
- ).crawl_url(api_request)
+ result = WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)
# In single-page mode, JinaReader returns data immediately
assert result["status"] == "active"
@@ -1407,11 +1381,7 @@ class TestWebsiteCrawlCases:
},
)
- result = WebsiteService(
- providers=website_service_module.DatasourceProviderService(
- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
- )
- ).crawl_url(api_request)
+ result = WebsiteService(providers=mock_provider_service.return_value).crawl_url(api_request)
assert result["status"] == "active"
assert result["job_id"] == "tag-filter-job"
diff --git a/api/tests/unit_tests/core/tools/test_tool_manager.py b/api/tests/unit_tests/core/tools/test_tool_manager.py
index e94d01f521f..f675fab54e2 100644
--- a/api/tests/unit_tests/core/tools/test_tool_manager.py
+++ b/api/tests/unit_tests/core/tools/test_tool_manager.py
@@ -246,6 +246,10 @@ def _setup_list_providers_from_api_mocks(
@pytest.fixture(autouse=True)
def _reset_tool_manager_state():
+ from extensions.application_services.tools import resolve_tool_credentials
+
+ old_credential_resolver = ToolManager._credential_resolver
+ ToolManager.configure_credential_resolver(resolve_tool_credentials)
old_hardcoded = ToolManager._hardcoded_providers.copy()
old_loaded = ToolManager._builtin_providers_loaded
old_labels = ToolManager._builtin_tools_labels.copy()
@@ -254,6 +258,7 @@ def _reset_tool_manager_state():
try:
yield
finally:
+ ToolManager._credential_resolver = old_credential_resolver
ToolManager._hardcoded_providers = old_hardcoded
ToolManager._builtin_providers_loaded = old_loaded
ToolManager._builtin_tools_labels = old_labels
@@ -405,7 +410,9 @@ def test_get_tool_runtime_builtin_with_credentials_decrypts_and_forks(
encrypter = Mock()
encrypter.decrypt.return_value = {"api_key": "secret"}
cache = Mock()
- with patch("core.tools.tool_manager.create_provider_encrypter", return_value=(encrypter, cache)):
+ with patch(
+ "services.credentials.tool_runtime_adapters.create_provider_encrypter", return_value=(encrypter, cache)
+ ):
result = ToolManager.get_tool_runtime(
provider_type=ToolProviderType.BUILT_IN,
provider_id="time",
@@ -419,16 +426,16 @@ def test_get_tool_runtime_builtin_with_credentials_decrypts_and_forks(
assert runtime.credential_type == CredentialType.API_KEY
-@patch("core.tools.tool_manager.create_provider_encrypter")
-@patch("core.plugin.impl.oauth.OAuthHandler")
+@patch("services.credentials.tool_runtime_adapters.create_provider_encrypter")
+@patch("services.credentials.tool_runtime_adapters.OAuthHandler")
@patch(
"services.tools.builtin_tools_manage_service.BuiltinToolManageService.get_oauth_client",
return_value=PluginOAuthClientConfig("langgenius/time:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"client_id": "id"}),
)
-@patch("core.tools.tool_manager.time.time", return_value=1000)
+@patch("services.credentials.tool_runtime.time.time", return_value=1000)
@patch("core.helper.credential_utils.check_credential_policy_compliance")
@pytest.mark.parametrize("client_credentials", [None, {}, {"client_id": "id"}])
[email protected]("plugin_provider", [False, True])
[email protected]("credential_selection", ["builtin", "plugin-explicit", "plugin-default"])
def test_get_tool_runtime_builtin_refreshes_expired_oauth_credentials(
mock_check,
mock_time,
@@ -438,7 +445,7 @@ def test_get_tool_runtime_builtin_refreshes_expired_oauth_credentials(
monkeypatch: pytest.MonkeyPatch,
tool_database: _ToolDatabase,
client_credentials: dict[str, str] | None,
- plugin_provider: bool,
+ credential_selection: str,
):
tool = Mock()
tool.fork_tool_runtime.return_value = "runtime-tool"
@@ -452,6 +459,7 @@ def test_get_tool_runtime_builtin_refreshes_expired_oauth_credentials(
mock_get_oauth_client.return_value = (
PluginOAuthClientConfig(identifier, client_credentials) if client_credentials is not None else None
)
+ plugin_provider = credential_selection != "builtin"
if plugin_provider:
controller = PluginToolProviderController(
entity=ToolProviderEntityWithPlugin.model_validate(
@@ -496,20 +504,18 @@ def test_get_tool_runtime_builtin_refreshes_expired_oauth_credentials(
provider_id="time",
tool_name="weekday",
tenant_id=tenant_id,
- credential_id=provider_id if plugin_provider else None,
+ credential_id=provider_id if credential_selection == "plugin-explicit" else None,
)
assert result == "runtime-tool"
- assert builtin_provider.expires_at == refreshed.expires_at
- assert builtin_provider.encrypted_credentials == json.dumps({"token": "encrypted"})
tool_database.session.expire_all()
persisted = tool_database.session.get(BuiltinToolProvider, provider_id)
assert persisted is not None
assert persisted.expires_at == refreshed.expires_at
assert persisted.encrypted_credentials == json.dumps({"token": "encrypted"})
- cache.delete.assert_called_once()
refresh = mock_oauth_handler_cls.return_value.refresh_credentials
refresh.assert_called_once()
+ assert refresh.call_args.kwargs["user_id"] == builtin_provider.user_id
assert refresh.call_args.kwargs["system_credentials"] == (client_credentials or {})
expected_identifier = identifier if plugin_provider or client_credentials is not None else None
assert refresh.call_args.kwargs["plugin_unique_identifier"] == expected_identifier
@@ -539,15 +545,15 @@ def test_get_tool_runtime_builtin_maps_oauth_refresh_failure_to_credential_error
encrypter.decrypt.return_value = {"token": "expired"}
with (
patch.object(ToolManager, "get_builtin_provider", return_value=controller),
- patch("core.tools.tool_manager.create_provider_encrypter", return_value=(encrypter, Mock())),
- patch("core.tools.tool_manager.time.time", return_value=1000),
+ patch("services.credentials.tool_runtime_adapters.create_provider_encrypter", return_value=(encrypter, Mock())),
+ patch("services.credentials.tool_runtime.time.time", return_value=1000),
patch(
"services.tools.builtin_tools_manage_service.BuiltinToolManageService.get_oauth_client",
return_value=PluginOAuthClientConfig(
"langgenius/time:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"client_id": "id"}
),
),
- patch("core.plugin.impl.oauth.OAuthHandler") as oauth_handler_cls,
+ patch("services.credentials.tool_runtime_adapters.OAuthHandler") as oauth_handler_cls,
):
oauth_handler_cls.return_value.refresh_credentials.side_effect = ValueError("refresh token revoked")
with pytest.raises(ToolProviderCredentialValidationError, match="could not be refreshed"):
diff --git a/api/tests/unit_tests/extensions/test_ext_application_services.py b/api/tests/unit_tests/extensions/test_ext_application_services.py
index ab18ed81c9c..2f8aea35211 100644
--- a/api/tests/unit_tests/extensions/test_ext_application_services.py
+++ b/api/tests/unit_tests/extensions/test_ext_application_services.py
@@ -255,6 +255,10 @@ def test_init_app_registers_services_for_the_current_app(
ext_application_services.init_app(app)
+ from extensions.application_services.tools import resolve_tool_credentials
+
+ assert ext_application_services.ToolManager._credential_resolver is resolve_tool_credentials
+
with app.app_context():
services = ext_application_services.application_services()
assert services is app.extensions["application_services"]
diff --git a/api/tests/unit_tests/repositories/credentials/test_query_repository.py b/api/tests/unit_tests/repositories/credentials/test_query_repository.py
index 9bb0ffab543..2d1b2cf44e4 100644
--- a/api/tests/unit_tests/repositories/credentials/test_query_repository.py
+++ b/api/tests/unit_tests/repositories/credentials/test_query_repository.py
@@ -3,11 +3,12 @@
Tests visibility filtering and complete domain reads against real SQLite rows.
"""
+from dataclasses import replace
from datetime import datetime
from uuid import uuid4
import pytest
-from sqlalchemy import select
+from sqlalchemy import delete, select, update
from sqlalchemy.orm import Session, sessionmaker
from core.plugin.entities.plugin_daemon import CredentialType as TriggerCredentialType
@@ -449,3 +450,107 @@ class TestApplyVisibilityFilter:
)
assert sqlite_session.scalars(query).all() == []
+
+
+def runtime_repository(sqlite_session_factory: sessionmaker[Session]) -> CredentialQueryRepository:
+ with sqlite_session_factory.begin() as session:
+ for index, (identity, provider, default) in enumerate(
+ [
+ ("oldest", "provider", False),
+ ("default", "org/plugin/provider", True),
+ ("other", "other", True),
+ ]
+ ):
+ row = BuiltinToolProvider(
+ tenant_id="workspace",
+ user_id="owner",
+ provider=provider,
+ name=identity,
+ credential_type=TriggerCredentialType.OAUTH2,
+ encrypted_credentials='{"token":"old"}',
+ expires_at=1,
+ is_default=default,
+ )
+ row.id = identity
+ row.created_at = datetime(2026, 1, index + 1)
+ row.updated_at = datetime(2026, 1, 1)
+ session.add(row)
+ return CredentialQueryRepository(session_factory=sqlite_session_factory)
+
+
+def test_lookup_is_scoped_and_prefers_default(runtime_repository: CredentialQueryRepository) -> None:
+ providers = ("org/plugin/provider", "provider")
+ record = runtime_repository.get_tool_credential(workspace_id="workspace", providers=providers, credential_id=None)
+ assert record is not None
+ assert record.id == "default"
+ assert (
+ runtime_repository.get_tool_credential(workspace_id="other", providers=providers, credential_id="default")
+ is None
+ )
+ assert (
+ runtime_repository.get_tool_credential(workspace_id="workspace", providers=providers, credential_id="other")
+ is None
+ )
+ oldest = runtime_repository.get_tool_credential(
+ workspace_id="workspace", providers=("provider",), credential_id=None
+ )
+ assert oldest is not None
+ assert oldest.id == "oldest"
+
+
+def test_refresh_commits_for_a_detached_snapshot(
+ runtime_repository: CredentialQueryRepository, sqlite_session_factory: sessionmaker[Session]
+) -> None:
+ record = runtime_repository.get_tool_credential(
+ workspace_id="workspace", providers=("provider",), credential_id="oldest"
+ )
+ assert record is not None
+ assert runtime_repository.update_tool_credential_if_unchanged(
+ record=record, encrypted_credentials='{"token":"new"}', expires_at=500
+ )
+ with sqlite_session_factory() as session:
+ persisted = session.get(BuiltinToolProvider, record.id)
+ assert persisted is not None
+ assert persisted.encrypted_credentials == '{"token":"new"}'
+ assert persisted.expires_at == 500
+ assert persisted.updated_at > datetime(2026, 1, 1)
+ assert record.encrypted_credentials == '{"token":"old"}'
+ assert not runtime_repository.update_tool_credential_if_unchanged(
+ record=record, encrypted_credentials="stale", expires_at=600
+ )
+
+
+ ("field", "value"),
+ [
+ ("tenant_id", "other"),
+ ("provider", "other"),
+ ("user_id", "other"),
+ ("credential_type", TriggerCredentialType.API_KEY),
+ ("encrypted_credentials", "reauthorized"),
+ ("expires_at", 2),
+ ("deleted", None),
+ ],
+)
+def test_refresh_preserves_concurrent_changes(
+ runtime_repository: CredentialQueryRepository,
+ sqlite_session_factory: sessionmaker[Session],
+ field: str,
+ value: object,
+) -> None:
+ record = runtime_repository.get_tool_credential(
+ workspace_id="workspace", providers=("provider",), credential_id="oldest"
+ )
+ assert record is not None
+ with sqlite_session_factory.begin() as session:
+ statement = (
+ delete(BuiltinToolProvider) if field == "deleted" else update(BuiltinToolProvider).values({field: value})
+ )
+ session.execute(statement.where(BuiltinToolProvider.id == record.id))
+ assert not runtime_repository.update_tool_credential_if_unchanged(
+ record=record, encrypted_credentials="refresh", expires_at=500
+ )
+ assert not runtime_repository.update_tool_credential_if_unchanged(
+ record=replace(record, workspace_id="foreign"), encrypted_credentials="refresh", expires_at=500
+ )
diff --git a/api/tests/unit_tests/repositories/data_source/test_credential_repository.py b/api/tests/unit_tests/repositories/data_source/test_credential_repository.py
index e330315dbe2..f83ea6165cf 100644
--- a/api/tests/unit_tests/repositories/data_source/test_credential_repository.py
+++ b/api/tests/unit_tests/repositories/data_source/test_credential_repository.py
@@ -310,8 +310,14 @@ def test_get_oauth_client_config_returns_detached_tenant_and_system_rows(
assert record.system_credentials == {"client_secret": "system-secret"}
+ ("field", "value"),
+ [("encrypted_credentials", {"integration_secret": "concurrent"}), ("expires_at", 200), ("auth_type", "api-key")],
+)
def test_update_if_unchanged_detects_concurrent_change(
sqlite_session_factory: sessionmaker[Session],
+ field: str,
+ value: object,
) -> None:
repository = SQLAlchemyDatasourceCredentialRepository(session_factory=sqlite_session_factory)
with sqlite_session_factory.begin() as session:
@@ -332,8 +338,7 @@ def test_update_if_unchanged_detects_concurrent_change(
with sqlite_session_factory.begin() as session:
stored = session.scalar(select(DatasourceProvider).where(DatasourceProvider.id == credential_id))
assert stored is not None
- stored.encrypted_credentials = {"integration_secret": "concurrent"}
- stored.expires_at = 200
+ setattr(stored, field, value)
assert not repository.update_if_unchanged(
record=record,
diff --git a/api/tests/unit_tests/services/credentials/test_redis_refresh_lock.py b/api/tests/unit_tests/services/credentials/test_redis_refresh_lock.py
new file mode 100644
index 00000000000..febeec491dc
--- /dev/null
+++ b/api/tests/unit_tests/services/credentials/test_redis_refresh_lock.py
@@ -0,0 +1,89 @@
+"""Lease lifetime and failure behavior with the real redis-py Lock class."""
+
+from threading import Event, Thread, current_thread
+from unittest.mock import MagicMock, patch
+
+import pytest
+from redis.exceptions import LockNotOwnedError, RedisError
+from redis.lock import Lock
+
+from extensions.ext_redis import RedisClientWrapper
+from services.credentials import redis_refresh_lock
+from services.credentials.redis_refresh_lock import RedisCredentialRefreshLock
+from services.credentials.refresh import CredentialRefreshUnavailableError
+
+
+def test_lease_renews_during_blocking_refresh_and_releases_on_error(
+ redis_transport: tuple[RedisClientWrapper, MagicMock], monkeypatch: pytest.MonkeyPatch
+) -> None:
+ redis, _ = redis_transport
+ renewed = Event()
+ monkeypatch.setattr(redis_refresh_lock, "_RENEW_SECONDS", 0.001)
+ with (
+ patch.object(Lock, "acquire", autospec=True, return_value=True) as acquire,
+ patch.object(Lock, "reacquire", side_effect=renewed.set),
+ patch.object(Lock, "release") as release,
+ ):
+
+ def refresh() -> None:
+ with RedisCredentialRefreshLock(client=redis).acquire(
+ credential_type="tool", workspace_id="workspace", credential_id="credential"
+ ) as lease:
+ assert renewed.wait(timeout=1)
+ lease.ensure_owned()
+ release.assert_not_called()
+ raise ValueError("provider failed")
+
+ with pytest.raises(ValueError, match="provider failed"):
+ refresh()
+ release.assert_called_once()
+ lock = acquire.call_args.args[0]
+ assert lock.name == "oauth:credential-refresh:tool:workspace:credential"
+ assert lock.timeout == 60
+ assert lock.blocking_timeout == 30
+ assert lock.thread_local is False
+
+
[email protected]("error", [LockNotOwnedError("expired"), RedisError("unavailable")])
+def test_lost_heartbeat_fails_closed(
+ redis_transport: tuple[RedisClientWrapper, MagicMock], monkeypatch: pytest.MonkeyPatch, error: RedisError
+) -> None:
+ redis, _ = redis_transport
+ attempted = Event()
+ threads: list[Thread] = []
+
+ def fail_renewal() -> None:
+ threads.append(current_thread())
+ attempted.set()
+ raise error
+
+ monkeypatch.setattr(redis_refresh_lock, "_RENEW_SECONDS", 0.001)
+ with (
+ patch.object(Lock, "acquire", return_value=True),
+ patch.object(Lock, "reacquire", side_effect=fail_renewal),
+ patch.object(Lock, "release", side_effect=LockNotOwnedError("expired")),
+ ):
+ with RedisCredentialRefreshLock(client=redis).acquire(
+ credential_type="datasource", workspace_id="workspace", credential_id="credential"
+ ) as lease:
+ assert attempted.wait(timeout=1)
+ threads[0].join(timeout=1)
+ with pytest.raises(CredentialRefreshUnavailableError):
+ lease.ensure_owned()
+
+
[email protected]("error", [None, RedisError("unavailable")])
+def test_busy_or_unavailable_lock_never_enters_refresh(
+ redis_transport: tuple[RedisClientWrapper, MagicMock], error: RedisError | None
+) -> None:
+ redis, _ = redis_transport
+ with (
+ patch.object(Lock, "acquire", return_value=False, side_effect=error),
+ patch.object(Lock, "release") as release,
+ ):
+ with pytest.raises(CredentialRefreshUnavailableError):
+ with RedisCredentialRefreshLock(client=redis).acquire(
+ credential_type="tool", workspace_id="workspace", credential_id="credential"
+ ):
+ pytest.fail("Refresh must not run without its lease")
+ release.assert_not_called()
diff --git a/api/tests/unit_tests/services/credentials/test_tool_runtime.py b/api/tests/unit_tests/services/credentials/test_tool_runtime.py
new file mode 100644
index 00000000000..d5e45dd97e5
--- /dev/null
+++ b/api/tests/unit_tests/services/credentials/test_tool_runtime.py
@@ -0,0 +1,162 @@
+"""Refresh coordination with concrete ports and concurrent callers."""
+
+import json
+import time
+from collections.abc import Callable, Mapping
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass, field, replace
+from threading import Barrier
+
+import pytest
+
+from services.credentials.refresh import CredentialRefreshUnavailableError
+from services.credentials.tool_runtime import (
+ RefreshedToolCredential,
+ ToolCredentialNotFoundError,
+ ToolCredentialRefreshError,
+ ToolCredentialService,
+ ToolRuntimeCredential,
+)
+from tests.helpers.unit.credential_refresh import ThreadCredentialRefreshLock
+
+
+def _record(*, expires_at: int = 1) -> ToolRuntimeCredential:
+ return ToolRuntimeCredential(
+ "credential", "workspace", "owner", "provider", "oauth2", '{"token":"old"}', expires_at
+ )
+
+
+@dataclass
+class CredentialStore:
+ record: ToolRuntimeCredential | None
+ barrier: Barrier | None = None
+ writes: int = 0
+
+ def get_tool_credential(
+ self, *, workspace_id: str, providers: tuple[str, ...], credential_id: str | None
+ ) -> ToolRuntimeCredential | None:
+ record = self.record
+ if record is not None:
+ assert record.workspace_id == workspace_id
+ assert record.provider in providers
+ assert credential_id in (None, record.id)
+ if self.barrier is not None and credential_id is None:
+ self.barrier.wait(timeout=2)
+ return record
+
+ def update_tool_credential_if_unchanged(
+ self, *, record: ToolRuntimeCredential, encrypted_credentials: str, expires_at: int
+ ) -> bool:
+ if self.record != record:
+ return False
+ self.writes += 1
+ self.record = replace(record, encrypted_credentials=encrypted_credentials, expires_at=expires_at)
+ return True
+
+
+@dataclass
+class CredentialRuntime:
+ during_refresh: Callable[[], None] = lambda: None
+ refreshes: int = 0
+ invalidations: list[str] = field(default_factory=list)
+
+ def check_policy(self, record: ToolRuntimeCredential) -> None:
+ assert record.workspace_id == "workspace"
+
+ def decrypt(self, record: ToolRuntimeCredential) -> dict[str, object]:
+ return json.loads(record.encrypted_credentials or "{}")
+
+ def encrypt(self, record: ToolRuntimeCredential, credentials: Mapping[str, object]) -> str:
+ assert record.workspace_id == "workspace"
+ return json.dumps(credentials)
+
+ def refresh(self, record: ToolRuntimeCredential, credentials: Mapping[str, object]) -> RefreshedToolCredential:
+ assert record.workspace_id == "workspace"
+ assert credentials == {"token": "old"}
+ self.refreshes += 1
+ self.during_refresh()
+ return RefreshedToolCredential({"token": "new"}, int(time.time()) + 3600)
+
+ def invalidate(self, record: ToolRuntimeCredential) -> None:
+ self.invalidations.append(record.id)
+
+
+def _service(
+ store: CredentialStore, runtime: CredentialRuntime, lock: ThreadCredentialRefreshLock
+) -> ToolCredentialService:
+ return ToolCredentialService(credentials=store, runtime=runtime, refresh_lock=lock, providers=("provider",))
+
+
+def test_concurrent_and_sequential_calls_refresh_once() -> None:
+ store = CredentialStore(_record(), barrier=Barrier(4))
+ lock = ThreadCredentialRefreshLock()
+ runtime = CredentialRuntime(during_refresh=lambda: lock.wait_for_callers(4))
+ service = _service(store, runtime, lock)
+ with ThreadPoolExecutor(max_workers=4) as pool:
+ futures = [pool.submit(service.resolve, workspace_id="workspace", credential_id=None) for _ in range(4)]
+ assert [future.result(timeout=3).credentials for future in futures] == [{"token": "new"}] * 4
+ assert service.resolve(workspace_id="workspace", credential_id="credential").credentials == {"token": "new"}
+ assert runtime.refreshes == store.writes == 1
+ assert runtime.invalidations == ["credential"]
+ assert lock.keys == [("tool", "workspace", "credential")] * 4
+
+
[email protected]("expires_at", [-1, int(time.time()) + 3600])
+def test_valid_credentials_do_not_acquire_a_lock(expires_at: int) -> None:
+ store, runtime, lock = (
+ CredentialStore(_record(expires_at=expires_at)),
+ CredentialRuntime(),
+ ThreadCredentialRefreshLock(),
+ )
+ assert _service(store, runtime, lock).resolve(workspace_id="workspace", credential_id=None).credentials == {
+ "token": "old"
+ }
+ assert lock.keys == []
+ assert runtime.refreshes == store.writes == 0
+
+
[email protected]("failure", ["unavailable", "lost-before", "lost-after", "provider"])
+def test_refresh_failure_does_not_write_or_return_expired_credentials(failure: str) -> None:
+ store, runtime, lock = CredentialStore(_record()), CredentialRuntime(), ThreadCredentialRefreshLock()
+ lock.unavailable = failure == "unavailable"
+ lock.lost = failure == "lost-before"
+
+ def fail_refresh() -> None:
+ if failure == "lost-after":
+ lock.lost = True
+ if failure == "provider":
+ raise ToolCredentialRefreshError("provider failed")
+
+ runtime.during_refresh = fail_refresh
+ with pytest.raises((ToolCredentialRefreshError, CredentialRefreshUnavailableError)):
+ _service(store, runtime, lock).resolve(workspace_id="workspace", credential_id=None)
+ assert store.writes == 0
+ assert runtime.invalidations == []
+ assert runtime.refreshes == (1 if failure in {"lost-after", "provider"} else 0)
+
+
[email protected]("change", ["reauthorized", "expired", "deleted"])
+def test_refresh_cannot_overwrite_concurrent_changes(change: str) -> None:
+ record = _record()
+ store, runtime, lock = CredentialStore(record), CredentialRuntime(), ThreadCredentialRefreshLock()
+
+ def change_credential() -> None:
+ store.record = (
+ None
+ if change == "deleted"
+ else replace(
+ record,
+ encrypted_credentials='{"token":"replacement"}',
+ expires_at=-1 if change == "reauthorized" else 1,
+ )
+ )
+
+ runtime.during_refresh = change_credential
+ service = _service(store, runtime, lock)
+ if change == "reauthorized":
+ assert service.resolve(workspace_id="workspace", credential_id=None).credentials == {"token": "replacement"}
+ else:
+ with pytest.raises(ToolCredentialNotFoundError if change == "deleted" else ToolCredentialRefreshError):
+ service.resolve(workspace_id="workspace", credential_id=None)
+ assert store.writes == 0
+ assert runtime.invalidations == []
diff --git a/api/tests/unit_tests/services/credentials/test_tool_runtime_adapters.py b/api/tests/unit_tests/services/credentials/test_tool_runtime_adapters.py
new file mode 100644
index 00000000000..e48ef5ee396
--- /dev/null
+++ b/api/tests/unit_tests/services/credentials/test_tool_runtime_adapters.py
@@ -0,0 +1,115 @@
+"""OAuth cache entries are bounded and checked against the persisted snapshot."""
+
+import json
+from dataclasses import replace
+
+import pytest
+
+from core.helper.provider_cache import ToolProviderCredentialsCache
+from core.tools.entities.tool_entities import ToolProviderEntityWithPlugin
+from core.tools.plugin_tool.provider import PluginToolProviderController
+from services.credentials.tool_runtime import ToolRuntimeCredential
+from services.credentials.tool_runtime_adapters import ProviderToolCredentialRuntime
+
+
+class CredentialCacheRedis:
+ def __init__(self) -> None:
+ self.values: dict[str, bytes] = {}
+ self.ttls: dict[str, int] = {}
+
+ def get(self, key: str) -> bytes | None:
+ return self.values.get(key)
+
+ def setex(self, key: str, ttl: int, value: str) -> None:
+ self.values[key] = value.encode()
+ self.ttls[key] = ttl
+
+ def delete(self, key: str) -> None:
+ self.values.pop(key, None)
+ self.ttls.pop(key, None)
+
+
+def _controller() -> PluginToolProviderController:
+ schemas = [
+ {"name": name, "type": "secret-input", "label": {"en_US": name}} for name in ("access_token", "refresh_token")
+ ]
+ return PluginToolProviderController(
+ entity=ToolProviderEntityWithPlugin.model_validate(
+ {
+ "identity": {
+ "author": "org",
+ "name": "provider",
+ "description": {"en_US": "Provider"},
+ "icon": "icon.svg",
+ "label": {"en_US": "Provider"},
+ },
+ "credentials_schema": schemas,
+ "oauth_schema": {"credentials_schema": schemas, "client_schema": []},
+ }
+ ),
+ plugin_id="org/plugin",
+ plugin_unique_identifier="org/plugin:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ tenant_id="workspace",
+ )
+
+
[email protected]("invalidate_from", ["runtime", "management"])
+def test_oauth_cache_reuses_only_matching_snapshots_and_invalidates_both_entries(
+ monkeypatch: pytest.MonkeyPatch, invalidate_from: str
+) -> None:
+ redis = CredentialCacheRedis()
+ monkeypatch.setattr("core.helper.provider_cache.redis_client", redis)
+ decryptions: list[str] = []
+
+ def decrypt(workspace_id: str, value: str) -> str:
+ assert workspace_id == "workspace"
+ decryptions.append(value)
+ return f"plain:{value}"
+
+ monkeypatch.setattr("core.helper.encrypter.decrypt_token", decrypt)
+ runtime = ProviderToolCredentialRuntime(
+ controller=_controller(), provider_id="org/plugin/provider", console_api_url=""
+ )
+ old_credentials = {"access_token": "access-old", "refresh_token": "refresh-old"}
+ old = ToolRuntimeCredential(
+ "credential", "workspace", "owner", "org/plugin/provider", "oauth2", json.dumps(old_credentials), -1
+ )
+ legacy = ToolProviderCredentialsCache(tenant_id=old.workspace_id, provider=old.provider, credential_id=old.id)
+ legacy.set({"token": "stale-legacy-token"})
+ old_plaintext = {name: f"plain:{value}" for name, value in old_credentials.items()}
+ for _ in range(10):
+ assert runtime.decrypt(old) == old_plaintext
+ assert len(decryptions) == 2
+
+ new_credentials = {"access_token": "access-new", "refresh_token": "refresh-new"}
+ latest = replace(old, encrypted_credentials=json.dumps(new_credentials))
+ new_plaintext = {name: f"plain:{value}" for name, value in new_credentials.items()}
+ assert runtime.decrypt(latest) == new_plaintext
+ assert len(decryptions) == 4
+
+ # A reader of the old snapshot may finish its cache write after a refresh.
+ runtime._cache(old).set(old_plaintext)
+ assert runtime.decrypt(latest) == new_plaintext
+ assert len(decryptions) == 6
+ assert runtime.decrypt(latest) == new_plaintext
+ assert len(decryptions) == 6
+ assert len(redis.values) == 2
+ assert set(redis.ttls.values()) == {86400}
+ assert runtime.decrypt(replace(latest, credential_type="api-key")) == {"token": "stale-legacy-token"}
+
+ if invalidate_from == "runtime":
+ runtime.invalidate(latest)
+ else:
+ legacy.delete()
+ assert redis.values == {}
+ assert redis.ttls == {}
+
+
+def test_oauth_cache_ignores_a_malformed_snapshot(monkeypatch: pytest.MonkeyPatch) -> None:
+ redis = CredentialCacheRedis()
+ monkeypatch.setattr("core.helper.provider_cache.redis_client", redis)
+ cache = ToolProviderCredentialsCache(
+ tenant_id="workspace", provider="provider", credential_id="credential", fingerprint="current"
+ )
+ redis.setex(cache.cache_key, 86400, '{"fingerprint":"current","credentials":[]}')
+ assert cache.get() is None
diff --git a/api/tests/unit_tests/services/data_source/test_credential_gateway.py b/api/tests/unit_tests/services/data_source/test_credential_gateway.py
index a14669f1726..c7f1f37f7fc 100644
--- a/api/tests/unit_tests/services/data_source/test_credential_gateway.py
+++ b/api/tests/unit_tests/services/data_source/test_credential_gateway.py
@@ -1,7 +1,10 @@
+import time
from collections.abc import Mapping
+from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass, field, replace
from datetime import datetime, timedelta
-from typing import cast
+from threading import Barrier, local
+from typing import Never, cast, override
import pytest
from sqlalchemy.orm import Session, sessionmaker
@@ -20,6 +23,7 @@ from core.plugin.impl.oauth import OAuthHandler
from models.oauth import DatasourceOauthParamConfig, DatasourceOauthTenantParamConfig
from models.provider_ids import DatasourceProviderID
from repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository
+from services.credentials.refresh import CredentialRefreshUnavailableError
from services.data_source.credential_adapters import (
OAuthDatasourceCredentialRefresher,
PluginDatasourceCredentialCodec,
@@ -35,7 +39,9 @@ from services.data_source.credential_gateway import (
TrustedStoredDatasourceCredentialGateway,
)
from services.data_source.entities.credential import DatasourceCredentialRecord
+from services.data_source.provider_service import DatasourceProviderService
from services.plugin.dispatch_identity import PluginOAuthClientConfig
+from tests.helpers.unit.credential_refresh import ThreadCredentialRefreshLock
def _record(
@@ -77,7 +83,7 @@ class ScriptedCredentialStore:
) -> DatasourceCredentialRecord | None:
self.events.append("read")
self.get_calls.append((workspace_id, actor_id, credential_id, provider, plugin_id))
- return self.records.pop(0)
+ return self.records.pop(0) if len(self.records) > 1 else self.records[0]
def update_if_unchanged(
self,
@@ -280,6 +286,7 @@ def _gateway(
credentials=store,
codec=ReversibleCredentialCodec(),
refresher=refresh,
+ refresh_lock=ThreadCredentialRefreshLock(),
now=lambda: 100,
),
refresh,
@@ -309,6 +316,7 @@ def test_trusted_stored_source_resolver_uses_distinct_owner_chain_port() -> None
credentials=catalog,
codec=ReversibleCredentialCodec(),
refresher=RecordingCredentialRefresher(),
+ refresh_lock=ThreadCredentialRefreshLock(),
now=lambda: 100,
)
@@ -339,6 +347,7 @@ def test_trusted_stored_source_resolver_fails_closed_for_missing_credential() ->
credentials=StoredCredentialCatalog(None),
codec=ReversibleCredentialCodec(),
refresher=RecordingCredentialRefresher(),
+ refresh_lock=ThreadCredentialRefreshLock(),
now=lambda: 100,
)
@@ -361,6 +370,7 @@ def test_trusted_stored_source_refreshes_with_persisted_credential_owner() -> No
credentials=catalog,
codec=ReversibleCredentialCodec(),
refresher=refresher,
+ refresh_lock=ThreadCredentialRefreshLock(),
now=lambda: 100,
)
@@ -396,7 +406,7 @@ def test_refresh_runs_between_read_and_conditional_update() -> None:
assert _resolve(gateway) == {"integration_secret": "refreshed"}
assert refresher.calls == [("workspace-1", "actor-1", record, {"integration_secret": "plain:encrypted"})]
assert store.updates == [(record, {"integration_secret": "cipher:refreshed"}, 500)]
- assert events == ["read", "refresh", "update"]
+ assert events == ["read", "read", "refresh", "update"]
def test_concurrent_refresh_uses_newer_non_expiring_snapshot() -> None:
@@ -426,6 +436,175 @@ def test_concurrent_refresh_fails_closed_if_latest_snapshot_is_still_expired() -
_resolve(gateway)
+def test_actor_stored_and_provider_callers_share_one_refresh() -> None:
+ class ReaderState(local):
+ first = True
+
+ class SharedCatalog:
+ record = _record(expires_at=1)
+ barrier = Barrier(3)
+ reader = ReaderState()
+ writes = 0
+
+ def _read(
+ self, workspace_id: str, credential_id: str | None, provider: str, plugin_id: str
+ ) -> DatasourceCredentialRecord:
+ record = self.record
+ assert (workspace_id, credential_id, provider, plugin_id) == (
+ record.workspace_id,
+ record.id,
+ record.provider,
+ record.plugin_id,
+ )
+ if self.reader.first:
+ self.reader.first = False
+ self.barrier.wait(timeout=2)
+ return record
+
+ def get_visible(
+ self, *, workspace_id: str, actor_id: str, credential_id: str, provider: str, plugin_id: str
+ ) -> DatasourceCredentialRecord:
+ assert actor_id == "actor-1"
+ return self._read(workspace_id, credential_id, provider, plugin_id)
+
+ def get_for_stored_document(
+ self,
+ *,
+ workspace_id: str,
+ dataset_id: str,
+ document_id: str,
+ credential_id: str | None,
+ provider: str,
+ plugin_id: str,
+ ) -> DatasourceCredentialRecord:
+ assert (dataset_id, document_id) == ("dataset-1", "document-1")
+ return self._read(workspace_id, credential_id, provider, plugin_id)
+
+ def get_provider_credential(
+ self, *, workspace_id: str, credential_id: str | None, provider: str, plugin_id: str
+ ) -> DatasourceCredentialRecord:
+ return self._read(workspace_id, credential_id, provider, plugin_id)
+
+ def list_provider_credentials(
+ self, *, workspace_id: str, provider: str, plugin_id: str
+ ) -> list[DatasourceCredentialRecord]:
+ return [self._read(workspace_id, self.record.id, provider, plugin_id)]
+
+ def get_oauth_client_config(self, *, workspace_id: str, provider: str, plugin_id: str) -> Never:
+ raise AssertionError(f"Unexpected OAuth lookup for {workspace_id}:{plugin_id}/{provider}")
+
+ def update_if_unchanged(
+ self, *, record: DatasourceCredentialRecord, encrypted_credentials: Mapping[str, object], expires_at: int
+ ) -> bool:
+ assert self.record == record
+ self.writes += 1
+ self.record = replace(record, encrypted_credentials=encrypted_credentials, expires_at=expires_at)
+ return True
+
+ class RotatingRefresher:
+ calls = 0
+
+ def refresh(
+ self,
+ *,
+ workspace_id: str,
+ actor_id: str,
+ record: DatasourceCredentialRecord,
+ credentials: Mapping[str, object],
+ ) -> RefreshedDatasourceCredential:
+ assert (workspace_id, record.id) == ("workspace-1", "credential-1")
+ assert actor_id in {"actor-1", "credential-owner-1"}
+ assert credentials == {"integration_secret": "plain:encrypted"}
+ self.calls += 1
+ lock.wait_for_callers(3)
+ return RefreshedDatasourceCredential({"integration_secret": "refreshed"}, int(time.time()) + 3600)
+
+ catalog, codec, refresher, lock = (
+ SharedCatalog(),
+ ReversibleCredentialCodec(),
+ RotatingRefresher(),
+ ThreadCredentialRefreshLock(),
+ )
+ actor = ActorAwareDatasourceCredentialGateway(
+ credentials=catalog, codec=codec, refresher=refresher, refresh_lock=lock
+ )
+ stored = TrustedStoredDatasourceCredentialGateway(
+ credentials=catalog, codec=codec, refresher=refresher, refresh_lock=lock
+ )
+ provider = DatasourceProviderService(
+ credentials=catalog,
+ codec=codec,
+ refresher=refresher,
+ refresh_lock=lock,
+ provider_manager=PluginDatasourceManager(),
+ )
+ with ThreadPoolExecutor(max_workers=3) as pool:
+ futures = [
+ pool.submit(_resolve, actor),
+ pool.submit(
+ stored.resolve_for_document,
+ workspace_id="workspace-1",
+ dataset_id="dataset-1",
+ document_id="document-1",
+ credential_id="credential-1",
+ provider="notion_datasource",
+ plugin_id="langgenius/notion_datasource",
+ ),
+ pool.submit(
+ provider.get_datasource_credentials,
+ "workspace-1",
+ "notion_datasource",
+ "langgenius/notion_datasource",
+ "credential-1",
+ ),
+ ]
+ results = [future.result(timeout=3) for future in futures]
+ assert all(result["integration_secret"] in {"refreshed", "plain:cipher:refreshed"} for result in results)
+ assert refresher.calls == catalog.writes == 1
+ assert lock.keys == [("datasource", "workspace-1", "credential-1")] * 3
+
+
[email protected]("failure", ["unavailable", "lost-before", "lost-after"])
+def test_refresh_lease_failure_never_writes_or_returns_expired_data(failure: str) -> None:
+ lock = ThreadCredentialRefreshLock()
+ lock.unavailable = failure == "unavailable"
+ lock.lost = failure == "lost-before"
+
+ class LosingRefresher(RecordingCredentialRefresher):
+ @override
+ def refresh(
+ self,
+ *,
+ workspace_id: str,
+ actor_id: str,
+ record: DatasourceCredentialRecord,
+ credentials: Mapping[str, object],
+ ) -> RefreshedDatasourceCredential:
+ result = super().refresh(
+ workspace_id=workspace_id, actor_id=actor_id, record=record, credentials=credentials
+ )
+ lock.lost = True
+ return result
+
+ store, refresher = ScriptedCredentialStore([_record(expires_at=1)]), LosingRefresher()
+ gateway = ActorAwareDatasourceCredentialGateway(
+ credentials=store, codec=ReversibleCredentialCodec(), refresher=refresher, refresh_lock=lock, now=lambda: 100
+ )
+ with pytest.raises(CredentialRefreshUnavailableError):
+ _resolve(gateway)
+ assert store.updates == []
+ assert len(refresher.calls) == (1 if failure == "lost-after" else 0)
+
+
+def test_visibility_is_rechecked_after_waiting_for_the_refresh_lock() -> None:
+ store = ScriptedCredentialStore([_record(expires_at=1), None])
+ gateway, refresher = _gateway(store)
+ with pytest.raises(DatasourceCredentialNotFoundError):
+ _resolve(gateway)
+ assert refresher.calls == []
+ assert store.updates == []
+
+
@pytest.mark.parametrize(
("auth_type", "secret_name"),
[("api-key", "api_secret"), ("oauth2", "integration_secret")],
diff --git a/api/tests/unit_tests/services/data_source/test_provider_service.py b/api/tests/unit_tests/services/data_source/test_provider_service.py
index fe6dc2a054f..38618d4e1ec 100644
--- a/api/tests/unit_tests/services/data_source/test_provider_service.py
+++ b/api/tests/unit_tests/services/data_source/test_provider_service.py
@@ -17,8 +17,10 @@ from models.provider_ids import DatasourceProviderID
from repositories.credentials.query_repository import CredentialQueryRepository
from repositories.data_source.credential_repository import SQLAlchemyDatasourceCredentialRepository
from services.data_source import provider_service as service_module
+from services.data_source.credential_gateway import DatasourceCredentialRefreshError, RefreshedDatasourceCredential
from services.data_source.provider_service import DatasourceProviderService, get_current_user
from services.plugin.dispatch_identity import PluginOAuthClientConfig
+from tests.helpers.unit.credential_refresh import build_datasource_provider_service
_PLUGIN_IDENTIFIER = f"org/plug:1.0.0@{'a' * 32}"
@@ -105,7 +107,7 @@ class TestDatasourceProviderService:
@pytest.fixture
def service(self, sqlite_session: Session):
- return DatasourceProviderService(
+ return build_datasource_provider_service(
credentials=SQLAlchemyDatasourceCredentialRepository(
session_factory=sessionmaker(bind=sqlite_session.get_bind(), expire_on_commit=False)
)
@@ -132,9 +134,10 @@ class TestDatasourceProviderService:
patch("core.plugin.impl.base._httpx_client.stream", side_effect=lambda **kw: httpx.stream(**kw)),
patch("httpx.request") as mock_httpx,
patch("services.data_source.provider_service.encrypter") as mock_enc,
+ patch("services.data_source.credential_adapters.encrypter", new=mock_enc),
patch("services.data_source.provider_service.redis_client") as mock_redis,
patch("services.data_source.provider_service.generate_incremental_name") as mock_genname,
- patch("services.data_source.provider_service.OAuthHandler") as mock_oauth,
+ patch("core.plugin.impl.oauth.OAuthHandler") as mock_oauth,
):
mock_enc.encrypt_token.return_value = "enc_tok"
mock_enc.decrypt_token.return_value = "dec_tok"
@@ -348,7 +351,6 @@ class TestDatasourceProviderService:
@pytest.mark.parametrize(
"oauth_client",
[
- None,
PluginOAuthClientConfig(_PLUGIN_IDENTIFIER, {}),
PluginOAuthClientConfig(_PLUGIN_IDENTIFIER, {"oc": "v"}),
],
@@ -359,12 +361,11 @@ class TestDatasourceProviderService:
persist(sqlite_session, p)
with (
patch("services.data_source.provider_service.get_current_user", return_value=mock_user),
- patch.object(
- service,
- "get_oauth_client",
+ patch(
+ "services.data_source.credential_adapters.PluginDatasourceOAuthClientResolver.resolve",
return_value=oauth_client,
),
- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"tok": "plain"}),
+ patch.object(service._codec, "decrypt", return_value={"tok": "plain"}),
):
service.get_datasource_credentials("t1", "prov", "org/plug")
sqlite_session.expire_all()
@@ -386,15 +387,13 @@ class TestDatasourceProviderService:
persist(sqlite_session, p)
with (
patch("services.data_source.provider_service.get_current_user", return_value=mock_user),
- patch("services.data_source.provider_service.OAuthHandler") as oauth_handler,
- patch.object(
- service,
- "get_oauth_client",
+ patch(
+ "services.data_source.credential_adapters.PluginDatasourceOAuthClientResolver.resolve",
return_value=PluginOAuthClientConfig("org/plug:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"oc": "v"}),
),
- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"tok": "plain"}),
+ patch.object(service._codec, "decrypt", return_value={"tok": "plain"}),
):
- oauth_handler.return_value.refresh_credentials.side_effect = RuntimeError("token endpoint failed")
+ self._oauth.return_value.refresh_credentials.side_effect = RuntimeError("token endpoint failed")
with pytest.raises(ValueError, match="provider prov"):
service.get_datasource_credentials("t1", "prov", "org/plug")
@@ -404,7 +403,7 @@ class TestDatasourceProviderService:
persist(sqlite_session, p)
with (
patch("services.data_source.provider_service.get_current_user", return_value=mock_user),
- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"k": "plain"}),
+ patch.object(service._codec, "decrypt", return_value={"k": "plain"}),
):
result = service.get_datasource_credentials("t1", "prov", "org/plug")
assert result == {"k": "plain"}
@@ -415,7 +414,7 @@ class TestDatasourceProviderService:
persist(sqlite_session, p)
with (
patch("services.data_source.provider_service.get_current_user", return_value=mock_user),
- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"k": "v"}),
+ patch.object(service._codec, "decrypt", return_value={"k": "v"}),
):
result = service.get_datasource_credentials("t1", "prov", "org/plug", credential_id="cred-id")
assert result == {"k": "v"}
@@ -442,7 +441,7 @@ class TestDatasourceProviderService:
super().__init__(**kwargs)
opened.append(self)
- def refresh(**_kwargs: object) -> tuple[dict[str, str], int]:
+ def refresh(**_kwargs: object) -> RefreshedDatasourceCredential:
assert opened
assert all(not session.in_transaction() for session in opened)
if concurrent_update:
@@ -452,19 +451,19 @@ class TestDatasourceProviderService:
.where(DatasourceProvider.id == row.id)
.values(encrypted_credentials={"token": "concurrent"}, expires_at=int(time.time()) + 3600)
)
- return {"token": "refreshed"}, int(time.time()) + 3600
+ return RefreshedDatasourceCredential({"token": "refreshed"}, int(time.time()) + 3600)
- service = DatasourceProviderService(
+ service = build_datasource_provider_service(
credentials=SQLAlchemyDatasourceCredentialRepository(
session_factory=sessionmaker(bind=sqlite_session.get_bind(), class_=ObservedSession)
)
)
monkeypatch.setattr(service_module, "get_current_user", lambda: mock_user)
- monkeypatch.setattr(service, "_refresh_datasource_credentials", refresh)
+ monkeypatch.setattr(service._refresher, "refresh", refresh)
monkeypatch.setattr(
- service,
- "decrypt_datasource_provider_credentials",
- lambda **kwargs: dict(kwargs["datasource_provider"].encrypted_credentials),
+ service._codec,
+ "decrypt",
+ lambda record: dict(record.encrypted_credentials),
)
result = service.get_datasource_credentials("t1", "prov", "org/plug")
assert result == {"token": "concurrent" if concurrent_update else "refreshed"}
@@ -489,12 +488,11 @@ class TestDatasourceProviderService:
persist(sqlite_session, p)
with (
patch("services.data_source.provider_service.get_current_user", return_value=mock_user),
- patch.object(
- service,
- "get_oauth_client",
+ patch(
+ "services.data_source.credential_adapters.PluginDatasourceOAuthClientResolver.resolve",
return_value=PluginOAuthClientConfig("org/plug:1.0.0@aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", {"oc": "v"}),
),
- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"t": "plain"}),
+ patch.object(service._codec, "decrypt", return_value={"t": "plain"}),
):
result = service.get_all_datasource_credentials_by_provider("t1", "prov", "org/plug")
assert len(result) == 1
@@ -518,11 +516,14 @@ class TestDatasourceProviderService:
with (
patch("services.data_source.provider_service.get_current_user", return_value=mock_user),
patch.object(
- service,
- "_refresh_datasource_credentials",
- side_effect=[ValueError("refresh failed"), ({"t": "enc"}, 9999)],
+ service._refresher,
+ "refresh",
+ side_effect=[
+ DatasourceCredentialRefreshError("failed-cred"),
+ RefreshedDatasourceCredential({"t": "plain"}, 9999),
+ ],
) as refresh_credentials,
- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"t": "plain"}),
+ patch.object(service._codec, "decrypt", return_value={"t": "plain"}),
):
result = service.get_all_datasource_credentials_by_provider("t1", "prov", "org/plug")
assert result == [{"t": "plain"}]
@@ -536,8 +537,8 @@ class TestDatasourceProviderService:
persist(sqlite_session, p)
with (
patch("services.data_source.provider_service.get_current_user", return_value=mock_user),
- patch.object(service, "_refresh_datasource_credentials") as refresh_credentials,
- patch.object(service, "decrypt_datasource_provider_credentials", return_value={"t": "plain"}),
+ patch.object(service._refresher, "refresh") as refresh_credentials,
+ patch.object(service._codec, "decrypt", return_value={"t": "plain"}),
):
result = service.get_all_datasource_credentials_by_provider("t1", "prov", "org/plug")
assert result == [{"t": "plain"}]
diff --git a/api/tests/unit_tests/services/data_source/test_website_service.py b/api/tests/unit_tests/services/data_source/test_website_service.py
index cde5d32a8ab..2be7df041ff 100644
--- a/api/tests/unit_tests/services/data_source/test_website_service.py
+++ b/api/tests/unit_tests/services/data_source/test_website_service.py
@@ -22,6 +22,7 @@ from services.data_source.website_service import (
WebsiteCrawlStatusApiRequest,
WebsiteService,
)
+from tests.helpers.unit.credential_refresh import build_datasource_provider_service
@dataclass(frozen=True)
@@ -112,11 +113,7 @@ def test_get_credentials_and_config_selects_plugin_id_and_key_firecrawl(monkeypa
service_instance.get_datasource_credentials.return_value = {"firecrawl_api_key": "k", "base_url": "b"}
monkeypatch.setattr(website_service_module, "DatasourceProviderService", MagicMock(return_value=service_instance))
- api_key, config = WebsiteService(
- providers=website_service_module.DatasourceProviderService(
- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
- )
- )._get_credentials_and_config("tenant-1", "firecrawl")
+ api_key, config = WebsiteService(providers=service_instance)._get_credentials_and_config("tenant-1", "firecrawl")
assert api_key == "k"
assert config["base_url"] == "b"
@@ -141,11 +138,7 @@ def test_get_credentials_and_config_selects_plugin_id_and_key_api_key(
service_instance.get_datasource_credentials.return_value = {"api_key": "enc-key", "base_url": "b"}
monkeypatch.setattr(website_service_module, "DatasourceProviderService", MagicMock(return_value=service_instance))
- api_key, config = WebsiteService(
- providers=website_service_module.DatasourceProviderService(
- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
- )
- )._get_credentials_and_config("tenant-1", provider)
+ api_key, config = WebsiteService(providers=service_instance)._get_credentials_and_config("tenant-1", provider)
assert api_key == "enc-key"
assert config["base_url"] == "b"
@@ -159,7 +152,7 @@ def test_get_credentials_and_config_selects_plugin_id_and_key_api_key(
def test_get_credentials_and_config_rejects_invalid_provider() -> None:
with pytest.raises(ValueError, match="Invalid provider"):
WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
)._get_credentials_and_config("tenant-1", "unknown")
@@ -187,11 +180,7 @@ def test_get_credentials_and_config_hits_unreachable_guard_branch(monkeypatch: p
monkeypatch.setattr(website_service_module, "DatasourceProviderService", MagicMock(return_value=service_instance))
with pytest.raises(ValueError, match="Invalid provider"):
- WebsiteService(
- providers=website_service_module.DatasourceProviderService(
- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
- )
- )._get_credentials_and_config("tenant-1", FlakyProvider()) # type: ignore[arg-type]
+ WebsiteService(providers=service_instance)._get_credentials_and_config("tenant-1", FlakyProvider()) # type: ignore[arg-type]
def test_get_decrypted_api_key_requires_api_key(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -222,7 +211,7 @@ def test_crawl_url_dispatches_by_provider(monkeypatch: pytest.MonkeyPatch) -> No
monkeypatch.setattr(WebsiteService, "_crawl_with_firecrawl", firecrawl_mock)
result = WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).crawl_url(api_request)
@@ -247,7 +236,7 @@ def test_crawl_url_dispatches_other_providers(monkeypatch: pytest.MonkeyPatch, p
monkeypatch.setattr(WebsiteService, method_name, impl_mock)
assert WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).crawl_url(api_request) == {"status": "active"}
@@ -260,7 +249,7 @@ def test_crawl_url_rejects_invalid_provider(monkeypatch: pytest.MonkeyPatch) ->
with pytest.raises(ValueError, match="Invalid provider"):
WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).crawl_url(api_request)
@@ -436,7 +425,7 @@ def test_get_crawl_status_dispatches(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(WebsiteService, "_get_firecrawl_status", firecrawl_status)
result = WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).get_crawl_status("job-1", "firecrawl")
@@ -446,7 +435,7 @@ def test_get_crawl_status_dispatches(monkeypatch: pytest.MonkeyPatch) -> None:
watercrawl_status = MagicMock(return_value={"status": "active", "job_id": "w"})
monkeypatch.setattr(WebsiteService, "_get_watercrawl_status", watercrawl_status)
assert WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).get_crawl_status("job-2", "watercrawl") == {"status": "active", "job_id": "w"}
@@ -455,7 +444,7 @@ def test_get_crawl_status_dispatches(monkeypatch: pytest.MonkeyPatch) -> None:
jinareader_status = MagicMock(return_value={"status": "active", "job_id": "j"})
monkeypatch.setattr(WebsiteService, "_get_jinareader_status", jinareader_status)
assert WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).get_crawl_status("job-3", "jinareader") == {"status": "active", "job_id": "j"}
@@ -466,7 +455,7 @@ def test_get_crawl_status_typed_rejects_invalid_provider(monkeypatch: pytest.Mon
monkeypatch.setattr(WebsiteService, "_get_credentials_and_config", MagicMock(return_value=("k", {})))
with pytest.raises(ValueError, match="Invalid provider"):
WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).get_crawl_status_typed(WebsiteCrawlStatusApiRequest(provider="bad", job_id="j"))
@@ -578,7 +567,7 @@ def test_get_jinareader_status_completed_formats_processed_items(monkeypatch: py
def test_get_crawl_url_data_dispatches_invalid_provider() -> None:
with pytest.raises(ValueError, match="Invalid provider"):
WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).get_crawl_url_data("job-1", "bad", "https://example.com", "tenant-1")
@@ -590,7 +579,7 @@ def test_get_crawl_url_data_hits_invalid_provider_branch_when_credentials_stubbe
monkeypatch.setattr(WebsiteService, "_get_credentials_and_config", MagicMock(return_value=("k", {})))
with pytest.raises(ValueError, match="Invalid provider"):
WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).get_crawl_url_data("job-1", object(), "u", "tenant-1") # type: ignore[arg-type]
@@ -610,7 +599,7 @@ def test_get_crawl_url_data_dispatches(monkeypatch: pytest.MonkeyPatch, provider
monkeypatch.setattr(WebsiteService, method_name, impl_mock)
result = WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).get_crawl_url_data("job-1", provider, "u", "tenant-1")
@@ -734,7 +723,7 @@ def test_get_scrape_url_data_dispatches_and_rejects_invalid_provider(monkeypatch
scrape_mock = MagicMock(return_value={"data": "x"})
monkeypatch.setattr(WebsiteService, "_scrape_with_firecrawl", scrape_mock)
assert WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).get_scrape_url_data("firecrawl", "u", "tenant-1", True) == {"data": "x"}
@@ -743,7 +732,7 @@ def test_get_scrape_url_data_dispatches_and_rejects_invalid_provider(monkeypatch
watercrawl_mock = MagicMock(return_value={"data": "y"})
monkeypatch.setattr(WebsiteService, "_scrape_with_watercrawl", watercrawl_mock)
assert WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).get_scrape_url_data("watercrawl", "u", "tenant-1", False) == {"data": "y"}
@@ -751,7 +740,7 @@ def test_get_scrape_url_data_dispatches_and_rejects_invalid_provider(monkeypatch
with pytest.raises(ValueError, match="Invalid provider"):
WebsiteService(
- providers=website_service_module.DatasourceProviderService(
+ providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
)
).get_scrape_url_data("jinareader", "u", "tenant-1", True)
diff --git a/api/tests/unit_tests/services/knowledge/indexing/adapters/test_estimate.py b/api/tests/unit_tests/services/knowledge/indexing/adapters/test_estimate.py
index 16efd0c6953..b52b1ceb845 100644
--- a/api/tests/unit_tests/services/knowledge/indexing/adapters/test_estimate.py
+++ b/api/tests/unit_tests/services/knowledge/indexing/adapters/test_estimate.py
@@ -17,6 +17,7 @@ from extensions.storage.storage_type import StorageType
from models.dataset import Dataset, DatasetProcessRule
from models.enums import CreatorUserRole, ProcessRuleMode
from models.model import UploadFile
+from services.credentials.refresh import CredentialRefreshUnavailableError
from services.knowledge.indexing.adapters.estimate import (
IndexingEstimateAdapter,
SQLAlchemyProcessRuleReader,
@@ -198,6 +199,7 @@ def test_estimate_selects_dataset_embedding_and_rejects_foreign_tenant(
@pytest.mark.parametrize(
("processor_error", "expected_error"),
[
+ (CredentialRefreshUnavailableError(), CredentialRefreshUnavailableError),
(LLMBadRequestError("provider unavailable"), IndexingEstimateProviderUnavailableError),
(ProviderTokenNotInitError("provider unavailable"), IndexingEstimateProviderUnavailableError),
(PluginDaemonClientSideError("provider unavailable"), IndexingEstimateProviderUnavailableError),
@@ -212,7 +214,7 @@ def test_estimate_translates_infrastructure_errors(
adapter = IndexingEstimateAdapter(session_factory=sqlite_session_factory)
with (
patch("services.knowledge.indexing.adapters.estimate.IndexProcessorFactory", side_effect=processor_error),
- pytest.raises(expected_error, match="provider unavailable|processor failed"),
+ pytest.raises(expected_error, match="provider unavailable|processor failed|temporarily unavailable"),
):
adapter.run(
"workspace-1",
diff --git a/api/tests/unit_tests/services/knowledge/indexing/adapters/test_sources.py b/api/tests/unit_tests/services/knowledge/indexing/adapters/test_sources.py
index 1ee3038b1b3..da387056de5 100644
--- a/api/tests/unit_tests/services/knowledge/indexing/adapters/test_sources.py
+++ b/api/tests/unit_tests/services/knowledge/indexing/adapters/test_sources.py
@@ -9,6 +9,7 @@ from core.rag.extractor.entity.datasource_type import DatasourceType
from core.rag.extractor.entity.extract_setting import UploadFileExtractionInput
from core.rag.extractor.notion_extractor import NotionExtractor
from extensions.application_services.data_sources import build_data_source_credentials
+from services.credentials.refresh import CredentialRefreshUnavailableError
from services.data_source.credential_gateway import (
DatasourceCredentialNotFoundError,
DatasourceCredentialRefreshError,
@@ -186,11 +187,21 @@ def test_stored_notion_resolution_uses_trusted_document_owner_chain() -> None:
("credential-1", {"integration_secret": "saved-secret"}, None, "saved-secret"),
("deleted-credential", {}, DatasourceCredentialNotFoundError(), "environment-secret"),
("credential-1", {}, DatasourceCredentialRefreshError("credential-1"), "environment-secret"),
+ ("credential-1", {}, CredentialRefreshUnavailableError(), "environment-secret"),
("credential-1", {}, None, "environment-secret"),
("credential-1", {"integration_secret": ""}, None, "environment-secret"),
("credential-1", {"integration_secret": 42}, None, "environment-secret"),
],
- ids=["legacy", "saved", "deleted", "refresh-failed", "missing-token", "empty-token", "invalid-token"],
+ ids=[
+ "legacy",
+ "saved",
+ "deleted",
+ "refresh-failed",
+ "refresh-unavailable",
+ "missing-token",
+ "empty-token",
+ "invalid-token",
+ ],
)
@pytest.mark.parametrize("integration_token", ["environment-secret", None, ""])
def test_stored_notion_preview_and_indexing_use_same_integration(
@@ -234,7 +245,12 @@ def test_stored_notion_preview_and_indexing_use_same_integration(
),
)
- if expected_token == "environment-secret" and not integration_token:
+ if isinstance(error, CredentialRefreshUnavailableError) and not integration_token:
+ with pytest.raises(CredentialRefreshUnavailableError):
+ resolver.resolve(source)
+ with pytest.raises(CredentialRefreshUnavailableError):
+ indexing_extractor()
+ elif expected_token == "environment-secret" and not integration_token:
with pytest.raises(SourceCredentialUnavailableError):
resolver.resolve(source)
with pytest.raises(ValueError, match="Must specify `integration_token`"):
diff --git a/api/tests/unit_tests/services/knowledge/indexing/test_estimate.py b/api/tests/unit_tests/services/knowledge/indexing/test_estimate.py
index 9cb7cc05066..de552f29827 100644
--- a/api/tests/unit_tests/services/knowledge/indexing/test_estimate.py
+++ b/api/tests/unit_tests/services/knowledge/indexing/test_estimate.py
@@ -8,6 +8,7 @@ from core.entities.knowledge_entities import IndexingEstimate
from core.rag.extractor.entity.datasource_type import DatasourceType
from core.rag.extractor.entity.extract_setting import ExtractSetting, UploadFileExtractionInput
from machinery.context import RequestContext
+from services.credentials.refresh import CredentialRefreshUnavailableError
from services.data_source.credential_gateway import DatasourceCredentialNotFoundError
from services.data_source.entities.notion_import import NotionPageType
from services.knowledge.dataset_access import AccessibleDataset, DatasetAccess, DatasetAccessDeniedError
@@ -333,6 +334,7 @@ def test_estimate_rejects_missing_document(fixture: ServiceFixture) -> None:
@pytest.mark.parametrize(
("source_error", "expected_error"),
[
+ (CredentialRefreshUnavailableError(), CredentialRefreshUnavailableError),
(DatasourceCredentialNotFoundError(), IndexingEstimateCredentialUnavailableError),
(IndexingInputSourceError("missing source"), EstimateSourceNotFoundError),
(UnsupportedStoredSourceError("unknown"), UnsupportedEstimateSourceError),
diff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline.py
index 88944418c83..c397af2789b 100644
--- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline.py
+++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline.py
@@ -46,6 +46,7 @@ from services.errors.rag_pipeline import RagPipelineResourceNotFoundError
from services.rag_pipeline import rag_pipeline as rag_pipeline_module
from services.rag_pipeline.rag_pipeline import RagPipelineService
from services.workflow_ref_service import WorkflowRef
+from tests.helpers.unit.credential_refresh import build_datasource_provider_service
def _make_service() -> RagPipelineService:
@@ -776,7 +777,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="website_crawl",
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -861,7 +862,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="online_document",
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -1043,9 +1044,7 @@ class TestRagPipelineServiceCases:
"d1",
True,
credential_query=mock.create_autospec(CredentialQuery, instance=True),
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
- credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
- ),
+ datasource_providers=mock_provider_service,
)
# 3. Assertions
@@ -1352,7 +1351,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="online_document",
is_published=False,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -1405,7 +1404,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type=DatasourceProviderType.ONLINE_DOCUMENT,
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -1462,7 +1461,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type=DatasourceProviderType.ONLINE_DRIVE,
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -1696,7 +1695,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="online_document",
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -1755,7 +1754,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="online_document",
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -1816,7 +1815,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="online_document",
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -1891,7 +1890,7 @@ class TestRagPipelineServiceCases:
"d1",
True,
credential_query=mock.create_autospec(CredentialQuery, instance=True),
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -1995,7 +1994,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="website_crawl",
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -2181,7 +2180,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="website_crawl",
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -2235,7 +2234,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="online_drive",
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -2290,7 +2289,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="online_document",
is_published=False,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -2452,7 +2451,7 @@ class TestRagPipelineServiceCases:
"d1",
True,
credential_query=mock.create_autospec(CredentialQuery, instance=True),
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -2489,7 +2488,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="online_document",
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -2510,7 +2509,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="online_document",
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -2563,7 +2562,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="online_document",
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -2610,7 +2609,7 @@ class TestRagPipelineServiceCases:
account=_make_account(),
datasource_type="online_document",
is_published=True,
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -2652,7 +2651,7 @@ class TestRagPipelineServiceCases:
"d1",
False,
credential_query=mock.create_autospec(CredentialQuery, instance=True),
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
@@ -2701,7 +2700,7 @@ class TestRagPipelineServiceCases:
"d1",
True,
credential_query=mock.create_autospec(CredentialQuery, instance=True),
- datasource_providers=rag_pipeline_module.DatasourceProviderService(
+ datasource_providers=build_datasource_provider_service(
credentials=create_autospec(DatasourceProviderCredentialStore, instance=True)
),
)
diff --git a/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_layer.py b/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_layer.py
index 2fe15eb5f03..e0aafb243fc 100644
--- a/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_layer.py
+++ b/dify-agent/tests/local/dify_agent/layers/dify_core_tools/test_layer.py
@@ -17,6 +17,11 @@ from tests.local.dify_agent.module_support import invoke_native_tool
[
(200, {"observation": "completed", "messages": [], "metadata": {}}, "completed"),
(429, {"code": "rate_limit", "message": "busy"}, "temporarily unavailable"),
+ (
+ 503,
+ {"code": "service_unavailable", "message": "Credential refresh is temporarily unavailable. Please retry."},
+ "temporarily unavailable",
+ ),
(404, {"code": "app_not_found", "message": "missing"}, "app context no longer exists"),
(403, {"code": "app_tenant_mismatch", "message": "invalid"}, "app context is invalid"),
(422, {"code": "agent_tool_credential_invalid", "message": "invalid"}, "check your tool provider credentials"),