RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Feat: data source test connection with unified connector validation (#14992)
天海蒼灆b85b908bb651
diff --git a/api/apps/restful_apis/connector_api.py b/api/apps/restful_apis/connector_api.pyindex 7a86876fc6..9a62c116e0 100644--- a/api/apps/restful_apis/connector_api.py+++ b/api/apps/restful_apis/connector_api.py@@ -28,7 +28,7 @@ from api.db import InputTypefrom api.db.services.connector_service import ConnectorService, SyncLogsServicefrom api.utils.api_utils import get_data_error_result, get_json_result, get_request_json, validate_requestfrom api.utils.pagination_utils import DEFAULT_PAGE, DEFAULT_PAGE_SIZE, validate_rest_api_page, validate_rest_api_page_size-from common.constants import RetCode, TaskStatus+from common.constants import FileSource, RetCode, TaskStatusfrom common.data_source.config import GOOGLE_DRIVE_WEB_OAUTH_REDIRECT_URI, GMAIL_WEB_OAUTH_REDIRECT_URI, BOX_WEB_OAUTH_REDIRECT_URI, DocumentSourcefrom common.data_source.google_util.constant import WEB_OAUTH_POPUP_TEMPLATE, GOOGLE_SCOPESfrom common.misc_utils import get_uuid@@ -180,102 +180,48 @@ def rm_connector(connector_id):@manager.route("/connectors/<connector_id>/test", methods=["POST"]) # noqa: F821@login_required+@validate_request("source")async def test_connector(connector_id):- """Validate connector configuration without persisting changes or triggering sync.-- For the REST API connector, this uses `RestAPIConnector.validate_config`- against the existing saved configuration. For BigQuery, it runs `SELECT 1`- plus a free dry-run of the configured base query under `maximum_bytes_billed`- so bad credentials, wrong location, or runaway scans surface before scheduled- syncs run (and incur cost).- """- if not ConnectorService.accessible(connector_id, current_user.id):+ """Validate connector configuration from the request body without persisting."""+ unsaved = connector_id in {source.value for source in FileSource if source.value}+ if not unsaved and not ConnectorService.accessible(connector_id, current_user.id):return _connector_auth_error(connector_id, current_user.id)+ from common.data_source import build_connector_for_sourcefrom common.data_source.exceptions import ConnectorMissingCredentialError, ConnectorValidationError- ok, conn = ConnectorService.get_by_id(connector_id)- if not ok:- return get_data_error_result(message="Can't find this Connector!")+ req = await get_request_json()+ source = req["source"]+ config = req.get("config") or {}+ if not isinstance(config, dict):+ return get_json_result(code=RetCode.ARGUMENT_ERROR, message="config must be an object.")- config = conn.config or {}- credentials = config.get("credentials") or {}-- if conn.source == DocumentSource.REST_API:- from common.data_source.rest_api_connector import RestAPIConnector-- try:- await asyncio.to_thread(- RestAPIConnector.validate_config,- config=config,- credentials=credentials,- )- except (ConnectorValidationError, ConnectorMissingCredentialError) as exc:- return get_json_result(- code=RetCode.DATA_ERROR,- message=str(exc),- data=False,- )- except Exception as exc:- logging.exception("REST API connector validation failed: %s", exc)- return get_json_result(- code=RetCode.SERVER_ERROR,- message="REST API connector validation failed, please check logs.",- data=False,- )-- return get_json_result(data=True)-- if conn.source == DocumentSource.BIGQUERY:- from common.data_source.bigquery_connector import BigQueryConnector-- def _validate_bigquery():- connector_kwargs = {- "project_id": config.get("project_id", ""),- "dataset_id": config.get("dataset_id") or None,- "table_id": config.get("table_id") or None,- "location": config.get("location") or None,- "query": config.get("query", ""),- "content_columns": config.get("content_columns", ""),- "metadata_columns": config.get("metadata_columns", ""),- "id_column": config.get("id_column") or None,- "timestamp_column": config.get("timestamp_column") or None,- "use_query_cache": config.get("use_query_cache", True),- }- if config.get("page_size") is not None:- connector_kwargs["page_size"] = int(config["page_size"])- if config.get("maximum_bytes_billed") is not None:- connector_kwargs["maximum_bytes_billed"] = int(config["maximum_bytes_billed"])- if config.get("job_timeout_ms") is not None:- connector_kwargs["job_timeout_ms"] = int(config["job_timeout_ms"])-- connector = BigQueryConnector(**connector_kwargs)- connector.load_credentials(credentials)- connector.validate_connector_settings()-- try:- await asyncio.to_thread(_validate_bigquery)- except (ConnectorValidationError, ConnectorMissingCredentialError) as exc:- return get_json_result(- code=RetCode.DATA_ERROR,- message=str(exc),- data=False,- )- except Exception as exc:- logging.exception("BigQuery connector validation failed: %s", exc)- return get_json_result(- code=RetCode.SERVER_ERROR,- message="BigQuery connector validation failed, please check logs.",- data=False,- )-- return get_json_result(data=True)+ if not unsaved:+ ok, conn = ConnectorService.get_by_id(connector_id)+ if ok and conn.tenant_id != current_user.id:+ return get_json_result(code=RetCode.PERMISSION_ERROR, message="You don't own this connector.")- return get_json_result(- code=RetCode.ARGUMENT_ERROR,- message="Test endpoint currently supports only REST API and BigQuery connectors.",- data=False,- )+ def _validate() -> None:+ connector = build_connector_for_source(source, config)+ connector.validate_connector_settings()++ try:+ await asyncio.to_thread(_validate)+ except (ConnectorValidationError, ConnectorMissingCredentialError) as exc:+ return get_json_result(+ code=RetCode.DATA_ERROR,+ message=str(exc),+ data=False,+ )+ except Exception as exc:+ logging.exception("Connector validation failed for %s: %s", connector_id, exc)+ return get_json_result(+ code=RetCode.SERVER_ERROR,+ message="Connector validation failed, please check logs.",+ data=False,+ )++ return get_json_result(data=True)WEB_FLOW_TTL_SECS = 15 * 60diff --git a/common/data_source/__init__.py b/common/data_source/__init__.pyindex 158e5450b5..f0865a3ead 100644--- a/common/data_source/__init__.py+++ b/common/data_source/__init__.py@@ -22,36 +22,100 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THESOFTWARE."""+from typing import Any++from common.constants import FileSource++from .airtable_connector import AirtableConnector+from .asana_connector import AsanaConnector+from .azure_blob_connector import AzureBlobConnector+from .bigquery_connector import BigQueryConnector+from .bitbucket.connector import BitbucketConnectorfrom .blob_connector import BlobStorageConnector-from .rss_connector import RSSConnector-from .slack_connector import SlackConnector-from .gmail_connector import GmailConnector-from .notion_connector import NotionConnector+from .box_connector import BoxConnectorfrom .confluence_connector import ConfluenceConnector+from .config import BlobType, DocumentSource+from .dingtalk_ai_table_connector import DingTalkAITableConnectorfrom .discord_connector import DiscordConnectorfrom .dropbox_connector import DropboxConnector+from .exceptions import (+ ConnectorMissingCredentialError,+ ConnectorValidationError,+ CredentialExpiredError,+ InsufficientPermissionsError,+ UnexpectedValidationError,+)+from .github.connector import GithubConnector+from .gitlab_connector import GitlabConnector+from .gmail_connector import GmailConnectorfrom .google_drive.connector import GoogleDriveConnector+from .imap_connector import ImapConnectorfrom .jira.connector import JiraConnector-from .sharepoint_connector import SharePointConnector+from .models import BasicExpertInfo, Document, ImageSection, TextSection+from .moodle_connector import MoodleConnector+from .notion_connector import NotionConnectorfrom .onedrive_connector import OneDriveConnectorfrom .outlook_connector import OutlookConnector+from .rdbms_connector import RDBMSConnector+from .rest_api_connector import RestAPIConnector+from .rss_connector import RSSConnectorfrom .salesforce_connector import SalesforceConnector-from .azure_blob_connector import AzureBlobConnector-from .teams_connector import TeamsConnector-from .moodle_connector import MoodleConnector-from .airtable_connector import AirtableConnector-from .dingtalk_ai_table_connector import DingTalkAITableConnector-from .asana_connector import AsanaConnector-from .imap_connector import ImapConnector-from .zendesk_connector import ZendeskConnectorfrom .seafile_connector import SeaFileConnector-from .rdbms_connector import RDBMSConnector-from .bigquery_connector import BigQueryConnector+from .sharepoint_connector import SharePointConnector+from .slack_connector import SlackConnector+from .teams_connector import TeamsConnectorfrom .webdav_connector import WebDAVConnector-from .rest_api_connector import RestAPIConnector-from .config import BlobType, DocumentSource-from .models import Document, TextSection, ImageSection, BasicExpertInfo-from .exceptions import ConnectorMissingCredentialError, ConnectorValidationError, CredentialExpiredError, InsufficientPermissionsError, UnexpectedValidationError+from .zendesk_connector import ZendeskConnector++CONNECTOR_BY_SOURCE: dict[str, type] = {+ FileSource.S3: BlobStorageConnector,+ FileSource.R2: BlobStorageConnector,+ FileSource.OCI_STORAGE: BlobStorageConnector,+ FileSource.GOOGLE_CLOUD_STORAGE: BlobStorageConnector,+ FileSource.RSS: RSSConnector,+ FileSource.CONFLUENCE: ConfluenceConnector,+ FileSource.NOTION: NotionConnector,+ FileSource.DISCORD: DiscordConnector,+ FileSource.GMAIL: GmailConnector,+ FileSource.DROPBOX: DropboxConnector,+ FileSource.GOOGLE_DRIVE: GoogleDriveConnector,+ FileSource.JIRA: JiraConnector,+ FileSource.SHAREPOINT: SharePointConnector,+ FileSource.SLACK: SlackConnector,+ FileSource.TEAMS: TeamsConnector,+ FileSource.WEBDAV: WebDAVConnector,+ FileSource.MOODLE: MoodleConnector,+ FileSource.BOX: BoxConnector,+ FileSource.AIRTABLE: AirtableConnector,+ FileSource.ASANA: AsanaConnector,+ FileSource.GITHUB: GithubConnector,+ FileSource.IMAP: ImapConnector,+ FileSource.ZENDESK: ZendeskConnector,+ FileSource.GITLAB: GitlabConnector,+ FileSource.BITBUCKET: BitbucketConnector,+ FileSource.SEAFILE: SeaFileConnector,+ FileSource.DINGTALK_AI_TABLE: DingTalkAITableConnector,+ FileSource.MYSQL: RDBMSConnector,+ FileSource.POSTGRESQL: RDBMSConnector,+ FileSource.REST_API: RestAPIConnector,+ FileSource.BIGQUERY: BigQueryConnector,+ FileSource.ONEDRIVE: OneDriveConnector,+ FileSource.OUTLOOK: OutlookConnector,+ FileSource.SALESFORCE: SalesforceConnector,+ FileSource.AZURE_BLOB: AzureBlobConnector,+}+++def build_connector_for_source(source: str, config: dict[str, Any]) -> Any:+ connector_cls = CONNECTOR_BY_SOURCE.get(source)+ if connector_cls is None:+ raise ConnectorValidationError(f"Unsupported data source type: {source}")+ if connector_cls is BlobStorageConnector:+ return connector_cls.build_connector(config, bucket_type=source)+ if connector_cls is RDBMSConnector:+ return connector_cls.build_connector(config, db_type=source)+ return connector_cls.build_connector(config)+__all__ = ["BlobStorageConnector",@@ -65,6 +129,10 @@ __all__ = ["GoogleDriveConnector","JiraConnector","SharePointConnector",+ "GithubConnector",+ "GitlabConnector",+ "BitbucketConnector",+ "BoxConnector","OneDriveConnector","OutlookConnector","SalesforceConnector",@@ -92,4 +160,6 @@ __all__ = ["WebDAVConnector","DingTalkAITableConnector","RestAPIConnector",+ "CONNECTOR_BY_SOURCE",+ "build_connector_for_source",]diff --git a/common/data_source/airtable_connector.py b/common/data_source/airtable_connector.pyindex 2ab471191c..ed35bbfa90 100644--- a/common/data_source/airtable_connector.py+++ b/common/data_source/airtable_connector.py@@ -7,7 +7,7 @@ import requestsfrom pyairtable import Api as AirtableApifrom common.data_source.config import AIRTABLE_CONNECTOR_SIZE_THRESHOLD, INDEX_BATCH_SIZE, DocumentSource-from common.data_source.exceptions import ConnectorMissingCredentialError+from common.data_source.exceptions import ConnectorMissingCredentialError, ConnectorValidationErrorfrom common.data_source.interfaces import LoadConnector, PollConnector, SlimConnectorWithPermSyncfrom common.data_source.models import (Document,@@ -81,10 +81,31 @@ class AirtableConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync)# -------------------------# Credentials# -------------------------+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "AirtableConnector":+ credentials = config.get("credentials") or {}+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ base_id=config.get("base_id"),+ table_name_or_id=config.get("table_name_or_id"),+ batch_size=batch_size,+ )+ connector.load_credentials({"airtable_access_token": credentials["airtable_access_token"]})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:self._airtable_client = AirtableApi(credentials["airtable_access_token"])return None+ def validate_connector_settings(self) -> None:+ if not self._airtable_client:+ raise ConnectorMissingCredentialError("Airtable credentials not loaded.")++ try:+ self.airtable_client.table(self.base_id, self.table_name_or_id).all(max_records=1)+ except Exception as e:+ raise ConnectorValidationError(f"Failed to validate Airtable connector settings: {e}") from e+@propertydef airtable_client(self) -> AirtableApi:if not self._airtable_client:diff --git a/common/data_source/asana_connector.py b/common/data_source/asana_connector.pyindex 2592888807..7e4375af0f 100644--- a/common/data_source/asana_connector.py+++ b/common/data_source/asana_connector.py@@ -6,6 +6,7 @@ from typing import Any, Dictimport asanaimport requestsfrom common.data_source.config import CONTINUE_ON_CONNECTOR_FAILURE, INDEX_BATCH_SIZE, DocumentSource+from common.data_source.exceptions import ConnectorMissingCredentialError, ConnectorValidationErrorfrom common.data_source.interfaces import LoadConnector, PollConnector, SlimConnectorWithPermSyncfrom common.data_source.models import Document, GenerateDocumentsOutput, GenerateSlimDocumentOutput, SecondsSinceUnixEpoch, SlimDocumentfrom common.data_source.utils import extract_size_bytes, get_file_ext@@ -329,6 +330,19 @@ class AsanaConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):self.size_threshold = Nonelogging.info(f"AsanaConnector initialized with workspace_id: {asana_workspace_id}")+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "AsanaConnector":+ credentials = config.get("credentials") or {}+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ config.get("asana_workspace_id"),+ config.get("asana_project_ids"),+ config.get("asana_team_id"),+ batch_size=batch_size,+ )+ connector.load_credentials({"asana_api_token_secret": credentials["asana_api_token_secret"]})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:self.api_token = credentials["asana_api_token_secret"]self.asana_client = AsanaAPI(@@ -340,6 +354,15 @@ class AsanaConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):logging.info("Asana credentials loaded and API client initialized")return None+ def validate_connector_settings(self) -> None:+ if not hasattr(self, "asana_client") or self.asana_client is None:+ raise ConnectorMissingCredentialError("Asana credentials not loaded.")++ try:+ self.asana_client.workspaces_api.get_workspace(self.workspace_id, {})+ except Exception as e:+ raise ConnectorValidationError(f"Failed to validate Asana connector settings: {e}") from e+def poll_source(self, start: SecondsSinceUnixEpoch, end: SecondsSinceUnixEpoch | None) -> GenerateDocumentsOutput:start_time = datetime.fromtimestamp(start, tz=timezone.utc).isoformat()end_time = datetime.fromtimestamp(end, tz=timezone.utc) if end is not None else Nonediff --git a/common/data_source/azure_blob_connector.py b/common/data_source/azure_blob_connector.pyindex 5ac88b10fc..498c3dd91c 100644--- a/common/data_source/azure_blob_connector.py+++ b/common/data_source/azure_blob_connector.py@@ -105,6 +105,18 @@ class AzureBlobConnector(CheckpointedConnectorWithPermSync, SlimConnectorWithPerself.auth_mode = (auth_mode or "").strip().lower()self._container_client = None+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "AzureBlobConnector":+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ batch_size=batch_size,+ prefix=config.get("prefix") or None,+ allow_images=bool(config.get("allow_images", False)),+ auth_mode=config.get("auth_mode") or None,+ )+ connector.load_credentials(config.get("credentials") or {})+ return connector+# ------------------------------------------------------------------# Auth# ------------------------------------------------------------------diff --git a/common/data_source/bigquery_connector.py b/common/data_source/bigquery_connector.pyindex 00489c5301..09ff99db48 100644--- a/common/data_source/bigquery_connector.py+++ b/common/data_source/bigquery_connector.py@@ -138,6 +138,32 @@ class BigQueryConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync)self._pending_sync_cursor_value: Any = Noneself._pending_sync_cursor_id: Any = None+ @classmethod+ def build_connector(cls, config: Dict[str, Any]) -> "BigQueryConnector":+ connector_kwargs: Dict[str, Any] = {+ "project_id": config.get("project_id", ""),+ "dataset_id": config.get("dataset_id") or None,+ "table_id": config.get("table_id") or None,+ "location": config.get("location") or None,+ "query": config.get("query", ""),+ "content_columns": config.get("content_columns", ""),+ "metadata_columns": config.get("metadata_columns", ""),+ "id_column": config.get("id_column") or None,+ "timestamp_column": config.get("timestamp_column") or None,+ "use_query_cache": config.get("use_query_cache", True),+ }+ if config.get("batch_size") is not None:+ connector_kwargs["batch_size"] = int(config["batch_size"])+ if config.get("page_size") is not None:+ connector_kwargs["page_size"] = int(config["page_size"])+ if config.get("maximum_bytes_billed") is not None:+ connector_kwargs["maximum_bytes_billed"] = int(config["maximum_bytes_billed"])+ if config.get("job_timeout_ms") is not None:+ connector_kwargs["job_timeout_ms"] = int(config["job_timeout_ms"])+ connector = cls(**connector_kwargs)+ connector.load_credentials(config.get("credentials") or {})+ return connector+# ------------------------------------------------------------------ ## Credentials & client# ------------------------------------------------------------------ #diff --git a/common/data_source/bitbucket/connector.py b/common/data_source/bitbucket/connector.pyindex 0e570f7786..798ecd9890 100644--- a/common/data_source/bitbucket/connector.py+++ b/common/data_source/bitbucket/connector.py@@ -82,6 +82,22 @@ class BitbucketConnector(self.email: str | None = Noneself.api_token: str | None = None+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "BitbucketConnector":+ credentials = config.get("credentials") or {}+ connector = cls(+ workspace=config.get("workspace"),+ repositories=config.get("repository_slugs"),+ projects=config.get("projects"),+ )+ connector.load_credentials(+ {+ "bitbucket_email": credentials.get("bitbucket_account_email"),+ "bitbucket_api_token": credentials.get("bitbucket_api_token"),+ }+ )+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:"""Load API token-based credentials.diff --git a/common/data_source/blob_connector.py b/common/data_source/blob_connector.pyindex d66e5a0993..ed6f3f1e1b 100644--- a/common/data_source/blob_connector.py+++ b/common/data_source/blob_connector.py@@ -76,6 +76,19 @@ class BlobStorageConnector(LoadConnector, PollConnector, FingerprintConnector):logging.info(f"Setting allow_images to {allow_images}.")self._allow_images = allow_images+ @classmethod+ def build_connector(cls, config: dict[str, Any], *, bucket_type: str) -> "BlobStorageConnector":+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ bucket_type=bucket_type,+ bucket_name=config["bucket_name"],+ prefix=config.get("prefix", ""),+ batch_size=batch_size,+ )+ connector.set_allow_images(config.get("allow_images", False))+ connector.load_credentials(config.get("credentials") or {})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:"""Load credentials"""logging.debug(f"Loading credentials for {self.bucket_name} of type {self.bucket_type}")diff --git a/common/data_source/box_connector.py b/common/data_source/box_connector.pyindex 51be0ffbcc..47d38260ac 100644--- a/common/data_source/box_connector.py+++ b/common/data_source/box_connector.py@@ -1,10 +1,11 @@"""Box connector"""+import jsonimport loggingfrom datetime import datetime, timezonefrom typing import Any, Generator-from box_sdk_gen import BoxClient+from box_sdk_gen import AccessToken, BoxClient, BoxOAuth, OAuthConfigfrom common.data_source.config import DocumentSource, INDEX_BATCH_SIZEfrom common.data_source.exceptions import (ConnectorMissingCredentialError,@@ -22,6 +23,29 @@ class BoxConnector(LoadConnector, PollConnector):self.use_marker = use_markerself.box_client: BoxClient | None = None+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "BoxConnector":+ credentials = config.get("credentials") or {}+ box_tokens = credentials.get("box_tokens")+ if not box_tokens:+ raise ConnectorMissingCredentialError("Box tokens are required.")+ token_payload = json.loads(box_tokens) if isinstance(box_tokens, str) else box_tokens+ auth = BoxOAuth(+ OAuthConfig(+ client_id=token_payload["client_id"],+ client_secret=token_payload["client_secret"],+ )+ )+ auth.token_storage.store(+ AccessToken(+ access_token=token_payload["access_token"],+ refresh_token=token_payload["refresh_token"],+ )+ )+ connector = cls(folder_id=config.get("folder_id", "0"))+ connector.load_credentials(auth)+ return connector+def load_credentials(self, auth: Any):self.box_client = BoxClient(auth=auth)return Nonediff --git a/common/data_source/confluence_connector.py b/common/data_source/confluence_connector.pyindex bb447ecdcd..af4c74a7d2 100644--- a/common/data_source/confluence_connector.py+++ b/common/data_source/confluence_connector.py@@ -1284,6 +1284,34 @@ class ConfluenceConnector(raise ConnectorMissingCredentialError("Confluence")return self._low_timeout_confluence_client+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "ConfluenceConnector":+ index_mode = (config.get("index_mode") or "everything").lower()+ space = ""+ page_id = ""+ index_recursively = False+ if index_mode == "space":+ space = (config.get("space") or "").strip()+ elif index_mode == "page":+ page_id = (config.get("page_id") or "").strip()+ index_recursively = bool(config.get("index_recursively", False))++ connector = cls(+ wiki_base=config["wiki_base"],+ is_cloud=config.get("is_cloud", True),+ space=space,+ page_id=page_id,+ index_recursively=index_recursively,+ )+ connector.set_credentials_provider(+ StaticCredentialsProvider(+ tenant_id=None,+ connector_name=DocumentSource.CONFLUENCE,+ credential_json=config.get("credentials") or {},+ )+ )+ return connector+def set_credentials_provider(self, credentials_provider: CredentialsProviderInterface) -> None:self.credentials_provider = credentials_providerdiff --git a/common/data_source/dingtalk_ai_table_connector.py b/common/data_source/dingtalk_ai_table_connector.pyindex 40dc44b61f..983e278ea9 100644--- a/common/data_source/dingtalk_ai_table_connector.py+++ b/common/data_source/dingtalk_ai_table_connector.py@@ -85,6 +85,18 @@ class DingTalkAITableConnector(LoadConnector, PollConnector, SlimConnectorWithPeconfig.region_id = "central"return NotableClient(config)+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "DingTalkAITableConnector":+ credentials = config.get("credentials") or {}+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ table_id=config.get("table_id"),+ operator_id=config.get("operator_id"),+ batch_size=batch_size,+ )+ connector.load_credentials({"access_token": credentials["access_token"]})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:"""Load DingTalk credentials.diff --git a/common/data_source/discord_connector.py b/common/data_source/discord_connector.pyindex 02deddc746..74bf0622e9 100644--- a/common/data_source/discord_connector.py+++ b/common/data_source/discord_connector.py@@ -248,6 +248,19 @@ class DiscordConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):self._discord_bot_token: str | None = Noneself.requested_start_date_string: str = start_date or ""+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "DiscordConnector":+ server_ids = config.get("server_ids")+ channel_names = config.get("channel_names")+ connector = cls(+ server_ids=server_ids.split(",") if server_ids else [],+ channel_names=channel_names.split(",") if channel_names else [],+ start_date=datetime(1970, 1, 1, tzinfo=timezone.utc).strftime("%Y-%m-%d"),+ batch_size=int(config.get("batch_size") or INDEX_BATCH_SIZE),+ )+ connector.load_credentials(config.get("credentials") or {})+ return connector+@propertydef discord_bot_token(self) -> str:if self._discord_bot_token is None:diff --git a/common/data_source/dropbox_connector.py b/common/data_source/dropbox_connector.pyindex 43ab08f4b0..71ae65cb26 100644--- a/common/data_source/dropbox_connector.py+++ b/common/data_source/dropbox_connector.py@@ -28,6 +28,13 @@ class DropboxConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):self.batch_size = batch_sizeself.dropbox_client: Dropbox | None = None+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "DropboxConnector":+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(batch_size=batch_size)+ connector.load_credentials(config.get("credentials") or {})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:"""Load Dropbox credentials"""access_token = credentials.get("dropbox_access_token")diff --git a/common/data_source/github/connector.py b/common/data_source/github/connector.pyindex 1e9be17d1b..76b7c69c9a 100644--- a/common/data_source/github/connector.py+++ b/common/data_source/github/connector.py@@ -358,6 +358,18 @@ class GithubConnector(CheckpointedConnectorWithPermSyncGH[GithubConnectorCheckpoself.include_issues = include_issuesself.github_client: Github | None = None+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "GithubConnector":+ credentials = config.get("credentials") or {}+ connector = cls(+ repo_owner=config.get("repository_owner"),+ repositories=config.get("repository_name"),+ include_prs=config.get("include_pull_requests", True),+ include_issues=config.get("include_issues", True),+ )+ connector.load_credentials({"github_access_token": credentials["github_access_token"]})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:# defaults to 30 items per page, can be set to as high as 100token = credentials["github_access_token"]diff --git a/common/data_source/gitlab_connector.py b/common/data_source/gitlab_connector.pyindex 2547ea54de..ff151f978b 100644--- a/common/data_source/gitlab_connector.py+++ b/common/data_source/gitlab_connector.py@@ -176,6 +176,24 @@ class GitlabConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):self.include_code_files = include_code_filesself.gitlab_client: gitlab.Gitlab | None = None+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "GitlabConnector":+ credentials = config.get("credentials") or {}+ connector = cls(+ project_owner=config.get("project_owner"),+ project_name=config.get("project_name"),+ include_mrs=config.get("include_mrs", False),+ include_issues=config.get("include_issues", False),+ include_code_files=config.get("include_code_files", False),+ )+ connector.load_credentials(+ {+ "gitlab_access_token": credentials.get("gitlab_access_token"),+ "gitlab_url": config.get("gitlab_url"),+ }+ )+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:self.gitlab_client = gitlab.Gitlab(credentials["gitlab_url"], private_token=credentials["gitlab_access_token"])return Nonediff --git a/common/data_source/gmail_connector.py b/common/data_source/gmail_connector.pyindex bdd03d6c14..4c0e582c14 100644--- a/common/data_source/gmail_connector.py+++ b/common/data_source/gmail_connector.py@@ -8,6 +8,12 @@ from common.data_source.config import INDEX_BATCH_SIZE, SLIM_BATCH_SIZE, Documenfrom common.data_source.google_util.auth import get_google_credsfrom common.data_source.google_util.constant import DB_CREDENTIALS_PRIMARY_ADMIN_KEY, MISSING_SCOPES_ERROR_STR, SCOPE_INSTRUCTIONS, USER_FIELDSfrom common.data_source.google_util.resource import get_admin_service, get_gmail_service+from common.data_source.exceptions import (+ ConnectorMissingCredentialError,+ ConnectorValidationError,+ CredentialExpiredError,+ InsufficientPermissionsError,+)from common.data_source.google_util.util import _execute_single_retrieval, execute_paginated_retrieval, clean_stringfrom common.data_source.interfaces import LoadConnector, PollConnector, SecondsSinceUnixEpoch, SlimConnectorWithPermSyncfrom common.data_source.models import BasicExpertInfo, Document, ExternalAccess, GenerateDocumentsOutput, GenerateSlimDocumentOutput, SlimDocument, TextSection@@ -168,6 +174,13 @@ class GmailConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):raise RuntimeError("Creds missing, should not call this property before calling load_credentials")return self._creds+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "GmailConnector":+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(batch_size=batch_size)+ connector.load_credentials(config.get("credentials") or {})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, str] | None:"""Load Gmail credentials."""primary_admin_email = credentials[DB_CREDENTIALS_PRIMARY_ADMIN_KEY]@@ -179,6 +192,28 @@ class GmailConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):)return new_creds_dict+ def validate_connector_settings(self) -> None:+ if self._creds is None:+ raise ConnectorMissingCredentialError("Gmail credentials not loaded.")++ try:+ gmail_service = get_gmail_service(self.creds, self.primary_admin_email)+ gmail_service.users().labels().list(+ userId=self.primary_admin_email,+ maxResults=1,+ ).execute()+ except HttpError as e:+ status_code = e.resp.status if e.resp else None+ if status_code == 401:+ raise CredentialExpiredError("Invalid or expired Gmail credentials (401).") from e+ if status_code == 403:+ raise InsufficientPermissionsError("Gmail app lacks required permissions (403).") from e+ raise ConnectorValidationError(f"Unexpected Gmail error (status={status_code}): {e}") from e+ except Exception as e:+ if MISSING_SCOPES_ERROR_STR in str(e):+ raise InsufficientPermissionsError("Gmail credentials are missing required scopes.") from e+ raise ConnectorValidationError(f"Unexpected error during Gmail validation: {e}") from e+def _get_all_user_emails(self) -> list[str]:"""Get all user emails for Google Workspace domain."""try:diff --git a/common/data_source/google_drive/connector.py b/common/data_source/google_drive/connector.pyindex 6fdae09bc5..298e846950 100644--- a/common/data_source/google_drive/connector.py+++ b/common/data_source/google_drive/connector.py@@ -192,6 +192,23 @@ class GoogleDriveConnector(SlimConnectorWithPermSync, CheckpointedConnectorWithPraise RuntimeError("Creds missing, should not call this property before calling load_credentials")return self._creds+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "GoogleDriveConnector":+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ include_shared_drives=config.get("include_shared_drives", False),+ include_my_drives=config.get("include_my_drives", False),+ include_files_shared_with_me=config.get("include_files_shared_with_me", False),+ shared_drive_urls=config.get("shared_drive_urls"),+ my_drive_emails=config.get("my_drive_emails"),+ shared_folder_urls=config.get("shared_folder_urls"),+ specific_user_emails=config.get("specific_user_emails"),+ batch_size=batch_size,+ )+ connector.set_allow_images(config.get("allow_images", False))+ connector.load_credentials(config.get("credentials") or {})+ return connector+# TODO: ensure returned new_creds_dict is actually persisted when this is called?def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:try:diff --git a/common/data_source/imap_connector.py b/common/data_source/imap_connector.pyindex 1d3560c2c5..f6e38315dd 100644--- a/common/data_source/imap_connector.py+++ b/common/data_source/imap_connector.py@@ -23,6 +23,7 @@ from common.data_source.interfaces import (CheckpointedConnectorWithPermSync,CredentialsConnector,CredentialsProviderInterface,+ StaticCredentialsProvider,)from common.data_source.models import (BasicExpertInfo,@@ -291,6 +292,22 @@ class ImapConnector(# impls for BaseConnector+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "ImapConnector":+ connector = cls(+ host=config.get("imap_host"),+ port=config.get("imap_port"),+ mailboxes=config.get("imap_mailbox"),+ )+ connector.set_credentials_provider(+ StaticCredentialsProvider(+ tenant_id=None,+ connector_name=DocumentSource.IMAP,+ credential_json=config.get("credentials") or {},+ )+ )+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:self._credentials = credentialsreturn Nonediff --git a/common/data_source/interfaces.py b/common/data_source/interfaces.pyindex 5c103f0603..18c5614cc7 100644--- a/common/data_source/interfaces.py+++ b/common/data_source/interfaces.py@@ -45,6 +45,10 @@ class LoadConnector(ABC):"""Validate connector settings"""pass+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "LoadConnector":+ raise NotImplementedError(f"{cls.__name__} must implement build_connector")+class PollConnector(ABC):"""Poll connector interface"""@@ -239,6 +243,10 @@ class BaseConnector(abc.ABC, Generic[CT]):Default is a no-op (always successful)."""+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "BaseConnector":+ raise NotImplementedError(f"{cls.__name__} must implement build_connector")+def validate_perm_sync(self) -> None:"""Permission-sync validation hook.diff --git a/common/data_source/jira/connector.py b/common/data_source/jira/connector.pyindex c562cf15e0..9f3e452019 100644--- a/common/data_source/jira/connector.py+++ b/common/data_source/jira/connector.py@@ -135,6 +135,26 @@ class JiraConnector(CheckpointedConnectorWithPermSync, SlimConnectorWithPermSync# Connector lifecycle helpers# -------------------------------------------------------------------------+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "JiraConnector":+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ jira_base_url=config["base_url"],+ project_key=config.get("project_key"),+ jql_query=config.get("jql_query"),+ batch_size=batch_size,+ include_comments=config.get("include_comments", True),+ include_attachments=config.get("include_attachments", False),+ labels_to_skip=config.get("labels_to_skip"),+ comment_email_blacklist=config.get("comment_email_blacklist"),+ scoped_token=config.get("scoped_token", False),+ attachment_size_limit=config.get("attachment_size_limit"),+ timezone_offset=config.get("timezone_offset"),+ time_buffer_seconds=config.get("time_buffer_seconds"),+ )+ connector.load_credentials(config.get("credentials") or {})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:"""Instantiate the Jira client using either an API token or username/password."""jira_url_for_client = self.jira_base_urldiff --git a/common/data_source/moodle_connector.py b/common/data_source/moodle_connector.pyindex 192155e03a..aef473f381 100644--- a/common/data_source/moodle_connector.py+++ b/common/data_source/moodle_connector.py@@ -67,6 +67,13 @@ class MoodleConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):for batch in batch_generator(generator, self.batch_size):yield batch+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "MoodleConnector":+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(moodle_url=config["moodle_url"], batch_size=batch_size)+ connector.load_credentials(config.get("credentials") or {})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> None:token = credentials.get("moodle_token")if not token:diff --git a/common/data_source/notion_connector.py b/common/data_source/notion_connector.pyindex 2972809294..7e9bdadb88 100644--- a/common/data_source/notion_connector.py+++ b/common/data_source/notion_connector.py@@ -666,6 +666,12 @@ class NotionConnector(LoadConnector, PollConnector):if slim_batch:yield slim_batch+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "NotionConnector":+ connector = cls(root_page_id=config["root_page_id"])+ connector.load_credentials(config.get("credentials") or {})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:"""Applies integration token to headers."""self.headers["Authorization"] = f"Bearer {credentials['notion_integration_token']}"diff --git a/common/data_source/onedrive_connector.py b/common/data_source/onedrive_connector.pyindex 0d2a614595..fdb2f336f2 100644--- a/common/data_source/onedrive_connector.py+++ b/common/data_source/onedrive_connector.py@@ -80,6 +80,13 @@ class OneDriveConnector(CheckpointedConnectorWithPermSync, SlimConnectorWithPermself._access_token: str | None = Noneself._tenant_id: str | None = None+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "OneDriveConnector":+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(batch_size=batch_size, folder_path=config.get("folder_path") or None)+ connector.load_credentials(config.get("credentials") or {})+ return connector+# ------------------------------------------------------------------# Auth# ------------------------------------------------------------------diff --git a/common/data_source/outlook_connector.py b/common/data_source/outlook_connector.pyindex 08aa210f52..6ed2abaef3 100644--- a/common/data_source/outlook_connector.py+++ b/common/data_source/outlook_connector.py@@ -120,6 +120,20 @@ class OutlookConnector(CheckpointedConnectorWithPermSync, SlimConnectorWithPermSself._access_token: str | None = Noneself._tenant_id: str | None = None+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "OutlookConnector":+ user_ids = config.get("user_ids")+ if isinstance(user_ids, str):+ user_ids = [item.strip() for item in user_ids.split(",") if item.strip()]+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ batch_size=batch_size,+ folder=config.get("folder") or _DEFAULT_FOLDER,+ user_ids=user_ids,+ )+ connector.load_credentials(config.get("credentials") or {})+ return connector+# ------------------------------------------------------------------# Auth# ------------------------------------------------------------------diff --git a/common/data_source/rdbms_connector.py b/common/data_source/rdbms_connector.pyindex 33551fb429..4614d95875 100644--- a/common/data_source/rdbms_connector.py+++ b/common/data_source/rdbms_connector.py@@ -130,6 +130,25 @@ class RDBMSConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):excluded = {self.id_column, self.timestamp_column}return [col for col in row_dict.keys() if col not in excluded]+ @classmethod+ def build_connector(cls, config: Dict[str, Any], *, db_type: str) -> "RDBMSConnector":+ default_port = 3306 if db_type == DatabaseType.MYSQL else 5432+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ db_type=db_type,+ host=config.get("host", "localhost"),+ port=int(config.get("port") or default_port),+ database=config.get("database", ""),+ query=config.get("query", ""),+ content_columns=config.get("content_columns", ""),+ metadata_columns=config.get("metadata_columns", ""),+ id_column=config.get("id_column") or None,+ timestamp_column=config.get("timestamp_column") or None,+ batch_size=batch_size,+ )+ connector.load_credentials(config.get("credentials") or {})+ return connector+def load_credentials(self, credentials: Dict[str, Any]) -> Dict[str, Any] | None:"""Load database credentials."""logging.debug(f"Loading credentials for {self.db_type} database: {self.database}")diff --git a/common/data_source/rest_api_connector.py b/common/data_source/rest_api_connector.pyindex 91d905089f..68279f205d 100644--- a/common/data_source/rest_api_connector.py+++ b/common/data_source/rest_api_connector.py@@ -428,6 +428,20 @@ class RestAPIConnector(LoadConnector, PollConnector):return cfg+ @classmethod+ def build_connector(cls, config: Dict[str, Any]) -> "RestAPIConnector":+ cfg = cls.parse_storage_config(config)+ connector = cls.from_parsed_config(cfg, max_pages=min(cfg.max_pages, 10))+ connector.load_credentials(config.get("credentials") or {})+ return connector++ def validate_connector_settings(self) -> None:+ try:+ logging.info("Validating REST API connector by fetching first page")+ _ = next(self._page_iter_for_validation())+ except StopIteration:+ pass+# -- LoadConnector / PollConnector interface -----------------------------def load_from_state(self) -> Generator[List[Document], None, None]:diff --git a/common/data_source/rss_connector.py b/common/data_source/rss_connector.pyindex 6fad756d73..fed9eee10d 100644--- a/common/data_source/rss_connector.py+++ b/common/data_source/rss_connector.py@@ -34,6 +34,13 @@ class RSSConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):self.credentials = credentials or {}return None+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "RSSConnector":+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(feed_url=config["feed_url"], batch_size=batch_size)+ connector.load_credentials(config.get("credentials") or {})+ return connector+def validate_connector_settings(self) -> None:self._validate_feed_url()if self.batch_size < 1:diff --git a/common/data_source/salesforce_connector.py b/common/data_source/salesforce_connector.pyindex 5e0c71f1c5..a9af3eb72c 100644--- a/common/data_source/salesforce_connector.py+++ b/common/data_source/salesforce_connector.py@@ -124,6 +124,20 @@ class SalesforceConnector(CheckpointedConnectorWithPermSync, SlimConnectorWithPeself._instance_url: str | None = Noneself._access_token: str | None = None+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "SalesforceConnector":+ objects = config.get("objects")+ if isinstance(objects, str):+ objects = [item.strip() for item in objects.split(",") if item.strip()]+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ batch_size=batch_size,+ objects=objects,+ api_version=config.get("api_version") or _DEFAULT_API_VERSION,+ )+ connector.load_credentials(config.get("credentials") or {})+ return connector+# ------------------------------------------------------------------# Auth# ------------------------------------------------------------------diff --git a/common/data_source/seafile_connector.py b/common/data_source/seafile_connector.pyindex c9ee59e083..90fa402898 100644--- a/common/data_source/seafile_connector.py+++ b/common/data_source/seafile_connector.py@@ -167,6 +167,20 @@ class SeaFileConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):)return resp+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "SeaFileConnector":+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ seafile_url=config["seafile_url"],+ batch_size=batch_size,+ include_shared=config.get("include_shared", True),+ sync_scope=config.get("sync_scope", SeafileSyncScope.ACCOUNT),+ repo_id=config.get("repo_id") or None,+ sync_path=config.get("sync_path") or None,+ )+ connector.load_credentials(config.get("credentials") or {})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:logger.debug("Loading credentials for SeaFile server %s", self.seafile_url)diff --git a/common/data_source/sharepoint_connector.py b/common/data_source/sharepoint_connector.pyindex 519c41c37f..67c6203cab 100644--- a/common/data_source/sharepoint_connector.py+++ b/common/data_source/sharepoint_connector.py@@ -49,6 +49,13 @@ class SharePointConnector(CheckpointedConnectorWithPermSync, SlimConnectorWithPe# -- credentials ---------------------------------------------------------+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "SharePointConnector":+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(batch_size=batch_size)+ connector.load_credentials(config.get("credentials") or {})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:"""Configure a Microsoft Graph client from app-only credentials.diff --git a/common/data_source/slack_connector.py b/common/data_source/slack_connector.pyindex fb2e235491..cdb653c71c 100644--- a/common/data_source/slack_connector.py+++ b/common/data_source/slack_connector.py@@ -14,9 +14,9 @@ from slack_sdk.errors import SlackApiErrorfrom slack_sdk.http_retry import ConnectionErrorRetryHandlerfrom slack_sdk.http_retry.builtin_interval_calculators import FixedValueRetryIntervalCalculator-from common.data_source.config import INDEX_BATCH_SIZE, SLACK_NUM_THREADS, ENABLE_EXPENSIVE_EXPERT_CALLS, _SLACK_LIMIT, FAST_TIMEOUT, MAX_RETRIES, MAX_CHANNELS_TO_LOG+from common.data_source.config import DocumentSource, INDEX_BATCH_SIZE, SLACK_NUM_THREADS, ENABLE_EXPENSIVE_EXPERT_CALLS, _SLACK_LIMIT, FAST_TIMEOUT, MAX_RETRIES, MAX_CHANNELS_TO_LOGfrom common.data_source.exceptions import ConnectorMissingCredentialError, ConnectorValidationError, CredentialExpiredError, InsufficientPermissionsError, UnexpectedValidationError-from common.data_source.interfaces import CheckpointedConnectorWithPermSync, CredentialsConnector, SlimConnectorWithPermSync+from common.data_source.interfaces import CheckpointedConnectorWithPermSync, CredentialsConnector, SlimConnectorWithPermSync, StaticCredentialsProviderfrom common.data_source.models import (BasicExpertInfo,ConnectorCheckpoint,@@ -450,6 +450,24 @@ class SlackConnector(def channels(self, channels: list[str] | None) -> None:self._channels = [channel.removeprefix("#") for channel in channels] if channels else None+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "SlackConnector":+ channels = config.get("channels")+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ channels=channels.split(",") if isinstance(channels, str) and channels else channels,+ channel_regex_enabled=bool(config.get("channel_regex_enabled", False)),+ batch_size=batch_size,+ )+ connector.set_credentials_provider(+ StaticCredentialsProvider(+ tenant_id=None,+ connector_name=DocumentSource.SLACK,+ credential_json=config.get("credentials") or {},+ )+ )+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:"""Load credentials"""raise NotImplementedError("Use set_credentials_provider with this connector.")diff --git a/common/data_source/teams_connector.py b/common/data_source/teams_connector.pyindex 2ca4604217..c31e03d31a 100644--- a/common/data_source/teams_connector.py+++ b/common/data_source/teams_connector.py@@ -56,6 +56,13 @@ class TeamsConnector(CheckpointedConnectorWithPermSync, SlimConnectorWithPermSyn# -- credentials ---------------------------------------------------------+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "TeamsConnector":+ batch_size = int(config.get("batch_size") or _SLIM_DOC_BATCH_SIZE)+ connector = cls(batch_size=batch_size)+ connector.load_credentials(config.get("credentials") or {})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:"""Configure a Microsoft Graph client from app-only credentials.diff --git a/common/data_source/webdav_connector.py b/common/data_source/webdav_connector.pyindex 4ba6bd3372..c205a6b3aa 100644--- a/common/data_source/webdav_connector.py+++ b/common/data_source/webdav_connector.py@@ -105,6 +105,18 @@ class WebDAVConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):logging.info(f"Setting allow_images to {allow_images}.")self._allow_images = allow_images+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "WebDAVConnector":+ batch_size = int(config.get("batch_size") or INDEX_BATCH_SIZE)+ connector = cls(+ base_url=config["base_url"],+ remote_path=config.get("remote_path", "/"),+ batch_size=batch_size,+ )+ connector.set_allow_images(config.get("allow_images", False))+ connector.load_credentials(config.get("credentials") or {})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:"""Load credentials and initialize WebDAV clientdiff --git a/common/data_source/zendesk_connector.py b/common/data_source/zendesk_connector.pyindex c740e994b9..57e5e41ec4 100644--- a/common/data_source/zendesk_connector.py+++ b/common/data_source/zendesk_connector.py@@ -318,6 +318,12 @@ class ZendeskConnector(SlimConnectorWithPermSync, CheckpointedConnector[ZendeskCself.content_tags: dict[str, str] = {}self.calls_per_minute = calls_per_minute+ @classmethod+ def build_connector(cls, config: dict[str, Any]) -> "ZendeskConnector":+ connector = cls(content_type=config.get("zendesk_content_type"))+ connector.load_credentials(config.get("credentials") or {})+ return connector+def load_credentials(self, credentials: dict[str, Any]) -> dict[str, Any] | None:# Subdomain is actually the whole URLsubdomain = credentials["zendesk_subdomain"].replace("https://", "").split(".zendesk.com")[0]diff --git a/test/testcases/restful_api/test_connector_routes_unit.py b/test/testcases/restful_api/test_connector_routes_unit.pyindex 4797e66cdd..7af3d0a5ad 100644--- a/test/testcases/restful_api/test_connector_routes_unit.py+++ b/test/testcases/restful_api/test_connector_routes_unit.py@@ -18,6 +18,7 @@ import asyncioimport importlib.utilimport jsonimport sys+from enum import StrEnumfrom pathlib import Pathfrom types import ModuleType, SimpleNamespace@@ -257,6 +258,12 @@ def _load_connector_app(monkeypatch):SCHEDULE="schedule",CANCEL="cancel",)++ class _FileSource(StrEnum):+ LOCAL = ""+ RSS = "rss"++ constants_mod.FileSource = _FileSourcemonkeypatch.setitem(sys.modules, "common.constants", constants_mod)config_mod = ModuleType("common.data_source.config")@@ -276,6 +283,26 @@ def _load_connector_app(monkeypatch):}monkeypatch.setitem(sys.modules, "common.data_source.google_util.constant", google_constants_mod)+ data_source_mod = ModuleType("common.data_source")++ def _stub_build_connector_for_source(_source, _config):+ raise NotImplementedError("patch build_connector_for_source in test")++ data_source_mod.build_connector_for_source = _stub_build_connector_for_source+ monkeypatch.setitem(sys.modules, "common.data_source", data_source_mod)++ data_source_exceptions_mod = ModuleType("common.data_source.exceptions")++ class _ConnectorMissingCredentialError(Exception):+ pass++ class _ConnectorValidationError(Exception):+ pass++ data_source_exceptions_mod.ConnectorMissingCredentialError = _ConnectorMissingCredentialError+ data_source_exceptions_mod.ConnectorValidationError = _ConnectorValidationError+ monkeypatch.setitem(sys.modules, "common.data_source.exceptions", data_source_exceptions_mod)+misc_mod = ModuleType("common.misc_utils")misc_mod.get_uuid = lambda: "uuid-from-helper"monkeypatch.setitem(sys.modules, "common.misc_utils", misc_mod)@@ -455,6 +482,33 @@ def test_connector_by_id_routes_reject_cross_tenant_access(monkeypatch):assert all(res["data"] is False for res in responses)assert touched == []+ class _FakeConnector:+ def validate_connector_settings(self):+ return None++ monkeypatch.setattr(+ sys.modules["common.data_source"],+ "build_connector_for_source",+ lambda source, config: _FakeConnector(),+ )+ monkeypatch.setattr(module.ConnectorService, "get_by_id", lambda _connector_id: (False, None))++ monkeypatch.setattr(+ module,+ "get_request_json",+ lambda: _AwaitableValue({"source": "rss", "config": {"feed_url": "https://example.com"}}),+ )+ ok_res = _run(module.test_connector("rss"))+ assert ok_res["data"] is True++ monkeypatch.setattr(+ module,+ "get_request_json",+ lambda: _AwaitableValue({"source": "rss", "config": "bad"}),+ )+ bad_config_res = _run(module.test_connector("rss"))+ assert bad_config_res["code"] == module.RetCode.ARGUMENT_ERROR+@pytest.mark.p2def test_connector_oauth_helper_functions(monkeypatch):diff --git a/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py b/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.pyindex 6f485af334..413c9771a8 100644--- a/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py+++ b/test/testcases/test_web_api/test_connector_app/test_connector_routes_unit.py@@ -18,6 +18,7 @@ import asyncioimport importlib.utilimport jsonimport sys+from enum import StrEnumfrom pathlib import Pathfrom types import ModuleType, SimpleNamespace@@ -253,6 +254,12 @@ def _load_connector_app(monkeypatch):AUTHENTICATION_ERROR=109,)constants_mod.TaskStatus = SimpleNamespace(SCHEDULE="schedule", CANCEL="cancel")++ class _FileSource(StrEnum):+ LOCAL = ""+ RSS = "rss"++ constants_mod.FileSource = _FileSourcemonkeypatch.setitem(sys.modules, "common.constants", constants_mod)config_mod = ModuleType("common.data_source.config")@@ -272,6 +279,26 @@ def _load_connector_app(monkeypatch):}monkeypatch.setitem(sys.modules, "common.data_source.google_util.constant", google_constants_mod)+ data_source_mod = ModuleType("common.data_source")++ def _stub_build_connector_for_source(_source, _config):+ raise NotImplementedError("patch build_connector_for_source in test")++ data_source_mod.build_connector_for_source = _stub_build_connector_for_source+ monkeypatch.setitem(sys.modules, "common.data_source", data_source_mod)++ data_source_exceptions_mod = ModuleType("common.data_source.exceptions")++ class _ConnectorMissingCredentialError(Exception):+ pass++ class _ConnectorValidationError(Exception):+ pass++ data_source_exceptions_mod.ConnectorMissingCredentialError = _ConnectorMissingCredentialError+ data_source_exceptions_mod.ConnectorValidationError = _ConnectorValidationError+ monkeypatch.setitem(sys.modules, "common.data_source.exceptions", data_source_exceptions_mod)+misc_mod = ModuleType("common.misc_utils")misc_mod.get_uuid = lambda: "uuid-from-helper"monkeypatch.setitem(sys.modules, "common.misc_utils", misc_mod)@@ -450,6 +477,33 @@ def test_connector_by_id_routes_reject_cross_tenant_access(monkeypatch):assert all(res["data"] is False for res in responses)assert touched == []+ class _FakeConnector:+ def validate_connector_settings(self):+ return None++ monkeypatch.setattr(+ sys.modules["common.data_source"],+ "build_connector_for_source",+ lambda source, config: _FakeConnector(),+ )+ monkeypatch.setattr(module.ConnectorService, "get_by_id", lambda _connector_id: (False, None))++ monkeypatch.setattr(+ module,+ "get_request_json",+ lambda: _AwaitableValue({"source": "rss", "config": {"feed_url": "https://example.com"}}),+ )+ ok_res = _run(module.test_connector("rss"))+ assert ok_res["data"] is True++ monkeypatch.setattr(+ module,+ "get_request_json",+ lambda: _AwaitableValue({"source": "rss", "config": "bad"}),+ )+ bad_config_res = _run(module.test_connector("rss"))+ assert bad_config_res["code"] == module.RetCode.ARGUMENT_ERROR+@pytest.mark.p2def test_connector_oauth_helper_functions(monkeypatch):diff --git a/web/src/components/dynamic-form.tsx b/web/src/components/dynamic-form.tsxindex d82599117f..c42510bd85 100644--- a/web/src/components/dynamic-form.tsx+++ b/web/src/components/dynamic-form.tsx@@ -157,6 +157,7 @@ export interface DynamicFormRef {submit: () => void;isDirty: () => boolean;getValues: (name?: string) => any;+ getFilteredValues: () => any;reset: (values?: any) => void;trigger: UseFormTrigger<any>;watch: (field: string, callback: (value: any) => void) => () => void;@@ -861,6 +862,7 @@ const DynamicForm = {},isDirty: () => form.formState.isDirty,getValues: form.getValues,+ getFilteredValues: () => filterActiveValues(form.getValues()),reset: (values?: T) => {if (values) {form.reset(values);diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex 1e0d62ab4c..931e1ddc5d 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -1800,6 +1800,10 @@ Example: Virtual Hosted Style`,restApiTestSuccess: 'REST API connector validated successfully.',restApiTestFailed:'REST API connector validation failed. Please check your configuration and logs.',+ dataSourceTestConnection: 'Test connection',+ dataSourceTestSuccess: 'Data source connection validated successfully.',+ dataSourceTestFailed:+ 'Data source connection validation failed. Please check your configuration and logs.',availableSourcesDescription: 'Select a data source to add',availableSources: 'Available sources',datasourceDescription: 'Manage your data source and connections',diff --git a/web/src/locales/zh.ts b/web/src/locales/zh.tsindex f3f35597e6..0ab839d53c 100644--- a/web/src/locales/zh.ts+++ b/web/src/locales/zh.ts@@ -1496,6 +1496,9 @@ NER:使用 spaCy NER 和基于规则的关键词提取来抽取 Entities 和 RavailableSourcesDescription: '选择要添加的数据源',availableSources: '可用数据源',datasourceDescription: '管理您的数据源和连接',+ dataSourceTestConnection: '测试连接',+ dataSourceTestSuccess: '数据源连接验证成功。',+ dataSourceTestFailed: '数据源连接验证失败,请检查配置和日志。',chatChannels: '聊天渠道',chatChannelsDescription: '管理您的聊天渠道机器人及凭证',channelEmptyTip: '暂未添加任何聊天渠道,请从下方选择一个进行连接。',diff --git a/web/src/pages/user-setting/data-source/add-datasource-modal.tsx b/web/src/pages/user-setting/data-source/add-datasource-modal.tsxindex 4aeda8a70f..7ba3536978 100644--- a/web/src/pages/user-setting/data-source/add-datasource-modal.tsx+++ b/web/src/pages/user-setting/data-source/add-datasource-modal.tsx@@ -14,10 +14,15 @@* limitations under the License.*/-import { DynamicForm, FormFieldConfig } from '@/components/dynamic-form';+import {+ DynamicForm,+ DynamicFormRef,+ FormFieldConfig,+} from '@/components/dynamic-form';+import { Button } from '@/components/ui/button';import { Modal } from '@/components/ui/modal/modal';import { IModalProps } from '@/interfaces/common';-import { useMemo } from 'react';+import { useMemo, useRef } from 'react';import { FieldValues } from 'react-hook-form';import { useTranslation } from 'react-i18next';import {@@ -27,6 +32,7 @@ import {getDataSourceFieldsWithExtras,mergeDataSourceFormValues,} from './constant';+import { useTestDataSource } from './hooks';import { IDataSorceInfo } from './interface';const AddDataSourceModal = ({@@ -37,6 +43,9 @@ const AddDataSourceModal = ({onOk,}: IModalProps<FieldValues> & { sourceData?: IDataSorceInfo }) => {const { t } = useTranslation();+ const formRef = useRef<DynamicFormRef>(null);+ const { loading: testLoading, handleTest } = useTestDataSource(formRef);+const fields = useMemo<FormFieldConfig[]>(() => {if (!sourceData) {return [];@@ -80,6 +89,7 @@ const AddDataSourceModal = ({footer={<div className="p-4"></div>}><DynamicForm.Root+ ref={formRef}fields={fields}onSubmit={(data) => {console.log(data);@@ -93,6 +103,15 @@ const AddDataSourceModal = ({hideModal?.();}}/>+ <Button+ type="button"+ variant="outline"+ onClick={handleTest}+ disabled={testLoading}+ loading={testLoading}+ >+ {t('setting.dataSourceTestConnection')}+ </Button><DynamicForm.SavingButtonsubmitLoading={loading || false}buttonText={t('common.confirm')}diff --git a/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx b/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsxindex 463f0dc7bd..bf41740e27 100644--- a/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx+++ b/web/src/pages/user-setting/data-source/data-source-detail-page/index.tsx@@ -29,9 +29,9 @@ import { isEqual } from 'lodash';import { useCallback, useEffect, useMemo, useRef, useState } from 'react';import { FieldValues } from 'react-hook-form';import { useTranslation } from 'react-i18next';+import { useSearchParams } from 'react-router';import {DataSourceFormDefaultValues,- DataSourceKey,getCommonExtraDefaultValues,getDataSourceFormBaseFields,getDataSourceFieldsWithExtras,@@ -49,6 +49,8 @@ import { DataSourceLogsTable } from './log-table';const SourceDetailPage = () => {const { t } = useTranslation();const formRef = useRef<DynamicFormRef>(null);+ const [searchParams] = useSearchParams();+ const connectorId = searchParams.get('id')!;const { data: detail } = useFetchDataSourceDetail();const { updateStatus, loading: statusUpdateLoading } =@@ -158,7 +160,10 @@ const SourceDetailPage = () => {}, [detail]);const { addLoading, handleAddOk } = useAddDataSource({ isEdit: true });- const { loading: testLoading, handleTest } = useTestDataSource();+ const { loading: testLoading, handleTest } = useTestDataSource(+ formRef,+ connectorId,+ );const onSubmit = useCallback(() => {formRef?.current?.submit();@@ -248,18 +253,15 @@ const SourceDetailPage = () => {/></div><div className="max-w-[1200px] flex justify-end gap-2">- {(detail?.source === DataSourceKey.REST_API ||- detail?.source === DataSourceKey.BIGQUERY) && (- <Button- type="button"- variant="outline"- onClick={handleTest}- disabled={testLoading}- loading={testLoading}- >- {t('setting.restApiTestConnection')}- </Button>- )}+ <Button+ type="button"+ variant="outline"+ onClick={handleTest}+ disabled={testLoading}+ loading={testLoading}+ >+ {t('setting.dataSourceTestConnection')}+ </Button><Buttontype="button"onClick={handlePrimaryAction}diff --git a/web/src/pages/user-setting/data-source/hooks.ts b/web/src/pages/user-setting/data-source/hooks.tsindex 181683b72b..fbad20f49e 100644--- a/web/src/pages/user-setting/data-source/hooks.ts+++ b/web/src/pages/user-setting/data-source/hooks.ts@@ -14,6 +14,7 @@* limitations under the License.*/+import { DynamicFormRef } from '@/components/dynamic-form';import message from '@/components/ui/message';import { RunningStatus } from '@/constants/knowledge';import { useSetModalState } from '@/hooks/common-hooks';@@ -28,7 +29,7 @@ import dataSourceService, {} from '@/services/data-source-service';import { useQuery, useQueryClient } from '@tanstack/react-query';import { t } from 'i18next';-import { useCallback, useMemo, useState } from 'react';+import { RefObject, useCallback, useMemo, useState } from 'react';import { useParams, useSearchParams } from 'react-router';import { DataSourceKey, useDataSourceInfo } from './constant';import {@@ -275,27 +276,34 @@ export const useDataSourceRebuild = () => {return { handleRebuild };};-export const useTestDataSource = () => {- const [currentQueryParameters] = useSearchParams();- const id = currentQueryParameters.get('id');+export const useTestDataSource = (+ formRef: RefObject<DynamicFormRef | null>,+ connectorId?: string,+) => {const [loading, setLoading] = useState(false);const handleTest = useCallback(async () => {- if (!id) return;+ const values = formRef.current?.getFilteredValues();+ const source = values?.source;+ if (!source) return;+setLoading(true);try {- const { data } = await testDataSource(id);+ const config =+ values?.config && typeof values.config === 'object'+ ? values.config+ : {};+ const { data } = await testDataSource(connectorId || source, {+ source,+ config,+ });if (data.code === 0) {- message.success(t('setting.restApiTestSuccess'));- } else {- message.error(data.message || t('setting.restApiTestFailed'));+ message.success(t('setting.dataSourceTestSuccess'));}- } catch {- message.error(t('setting.restApiTestFailed'));} finally {setLoading(false);}- }, [id]);+ }, [formRef, connectorId]);return { loading, handleTest };};diff --git a/web/src/services/data-source-service.ts b/web/src/services/data-source-service.tsindex 272bf6b7e9..0d3930204a 100644--- a/web/src/services/data-source-service.ts+++ b/web/src/services/data-source-service.ts@@ -50,8 +50,10 @@ export const getDataSourceLogs = (id: string, params?: any) =>export const featchDataSourceDetail = (id: string) =>request.get(api.dataSourceDetail(id));-export const testDataSource = (id: string) =>- request.post(api.dataSourceTest(id));+export const testDataSource = (+ id: string,+ data: { source: string; config?: Record<string, unknown> },+) => request.post(api.dataSourceTest(id), { data });export const startGoogleDriveWebAuth = (payload: {credentials: string;