RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(webdav): support custom CA certificates (#16901)
taek1053f4fcef4ffdd
diff --git a/common/data_source/webdav_connector.py b/common/data_source/webdav_connector.pyindex 5a8750d684..eb31563a70 100644--- a/common/data_source/webdav_connector.py+++ b/common/data_source/webdav_connector.py@@ -26,6 +26,7 @@ class WebDAVConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):base_url: str,remote_path: str = "/",batch_size: int = INDEX_BATCH_SIZE,+ ca_cert_path: str | None = None,) -> None:"""Initialize WebDAV connector@@ -33,6 +34,7 @@ class WebDAVConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):base_url: Base URL of the WebDAV server (e.g., "https://webdav.example.com")remote_path: Remote path to sync from (default: "/")batch_size: Number of documents per batch+ ca_cert_path: Optional path to a custom CA certificate bundle inside the container"""self.base_url = base_url.rstrip("/")if not remote_path:@@ -43,6 +45,9 @@ class WebDAVConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):remote_path = remote_path.rstrip("/")self.remote_path = remote_pathself.batch_size = batch_size+ if ca_cert_path is not None and not isinstance(ca_cert_path, str):+ raise ConnectorValidationError("WebDAV CA certificate path must be a string.")+ self.ca_cert_path = ca_cert_path.strip() if ca_cert_path else Noneself.client: Optional[WebDAVClient] = Noneself._allow_images: bool | None = Noneself.size_threshold: int | None = BLOB_STORAGE_SIZE_THRESHOLD@@ -112,6 +117,7 @@ class WebDAVConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):base_url=config["base_url"],remote_path=config.get("remote_path", "/"),batch_size=batch_size,+ ca_cert_path=config.get("ca_cert_path"),)connector.set_allow_images(config.get("allow_images", False))connector.load_credentials(config.get("credentials") or {})@@ -128,6 +134,7 @@ class WebDAVConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):Raises:ConnectorMissingCredentialError: If required credentials are missing+ ConnectorValidationError: If TLS verification cannot be configured"""logging.debug(f"Loading credentials for WebDAV server {self.base_url}")@@ -138,11 +145,20 @@ class WebDAVConnector(LoadConnector, PollConnector, SlimConnectorWithPermSync):raise ConnectorMissingCredentialError("WebDAV requires 'username' and 'password' credentials")try:- # Initialize WebDAV client- self.client = WebDAVClient(base_url=self.base_url, auth=(username, password))- except Exception as e:- logging.error(f"Failed to connect to WebDAV server: {e}")- raise ConnectorMissingCredentialError(f"Failed to authenticate with WebDAV server: {e}")+ # Omit `verify` unless a custom CA is configured so webdav4 keeps its secure default.+ client_options: dict[str, Any] = {}+ if self.ca_cert_path:+ client_options["verify"] = self.ca_cert_path+ logging.debug(f"Enabled custom TLS verification for WebDAV server {self.base_url}")++ self.client = WebDAVClient(+ base_url=self.base_url,+ auth=(username, password),+ **client_options,+ )+ except OSError as e:+ logging.error(f"Failed to configure TLS verification for WebDAV server: {e}")+ raise ConnectorValidationError(f"Failed to configure WebDAV TLS verification: {e}") from ereturn Nonediff --git a/internal/syncer/connector/webdav.go b/internal/syncer/connector/webdav.goindex fe8e8c43fc..012b41f77c 100644--- a/internal/syncer/connector/webdav.go+++ b/internal/syncer/connector/webdav.go@@ -19,6 +19,8 @@ package connectorimport ("bytes""context"+ "crypto/tls"+ "crypto/x509""encoding/xml""errors""fmt"@@ -75,6 +77,10 @@ func NewWebDAVConnector(config map[string]any) (*WebDAVConnector, error) {remotePath := normalizeWebDAVPath(stringConfig(config["remote_path"]))username := strings.TrimSpace(stringConfig(credentials["username"]))password := stringConfig(credentials["password"])+ httpClient, err := newWebDAVHTTPClient(strings.TrimSpace(stringConfig(config["ca_cert_path"])))+ if err != nil {+ return nil, err+ }connector := &WebDAVConnector{baseURL: baseURL,remotePath: remotePath,@@ -84,10 +90,8 @@ func NewWebDAVConnector(config map[string]any) (*WebDAVConnector, error) {password: password,sizeThreshold: webDAVSizeThreshold(),client: &webdavClient{- baseURL: baseURL,- httpClient: &http.Client{- Timeout: webdavRequestTimeout,- },+ baseURL: baseURL,+ httpClient: httpClient,},}connector.client.username = username@@ -97,6 +101,39 @@ func NewWebDAVConnector(config map[string]any) (*WebDAVConnector, error) {return connector, nil}+func newWebDAVHTTPClient(caCertPath string) (*http.Client, error) {+ client := &http.Client{Timeout: webdavRequestTimeout}+ if caCertPath == "" {+ return client, nil+ }++ caPEM, err := os.ReadFile(caCertPath)+ if err != nil {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("failed to read WebDAV CA certificate %q: %v", caCertPath, err)}+ }+ rootCAs, err := x509.SystemCertPool()+ if err != nil || rootCAs == nil {+ rootCAs = x509.NewCertPool()+ }+ if !rootCAs.AppendCertsFromPEM(caPEM) {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("WebDAV CA certificate %q contains no valid PEM certificates", caCertPath)}+ }++ defaultTransport, ok := http.DefaultTransport.(*http.Transport)+ if !ok {+ return nil, fmt.Errorf("unsupported default HTTP transport type %T", http.DefaultTransport)+ }+ transport := defaultTransport.Clone()+ if transport.TLSClientConfig == nil {+ transport.TLSClientConfig = &tls.Config{}+ } else {+ transport.TLSClientConfig = transport.TLSClientConfig.Clone()+ }+ transport.TLSClientConfig.RootCAs = rootCAs+ client.Transport = transport+ return client, nil+}+// Validate validates WebDAV settings and credentials by probing the remote path.func (c *WebDAVConnector) Validate(ctx context.Context) error {if c == nil {diff --git a/internal/syncer/connector/webdav_test.go b/internal/syncer/connector/webdav_test.goindex c171cdaa20..dcf5a8ec38 100644--- a/internal/syncer/connector/webdav_test.go+++ b/internal/syncer/connector/webdav_test.go@@ -20,11 +20,14 @@ import ("bytes""context""encoding/base64"+ "encoding/pem""errors""fmt""io""net/http""net/http/httptest"+ "os"+ "path/filepath""strings""testing")@@ -170,6 +173,82 @@ func webDAVTestConnector(t *testing.T, serverURL string, allowImages bool, batchreturn connector}+func TestNewWebDAVConnectorPreservesDefaultHTTPTransport(t *testing.T) {+ connector, err := NewWebDAVConnector(map[string]any{})+ if err != nil {+ t.Fatalf("NewWebDAVConnector failed: %v", err)+ }+ if connector.client.httpClient.Transport != nil {+ t.Fatalf("HTTP transport = %T, want nil default transport", connector.client.httpClient.Transport)+ }+}++func TestNewWebDAVConnectorUsesCustomCACertificate(t *testing.T) {+ server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusNoContent)+ }))+ defer server.Close()++ caCertPath := filepath.Join(t.TempDir(), "webdav-ca.pem")+ caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: server.Certificate().Raw})+ if err := os.WriteFile(caCertPath, caPEM, 0o600); err != nil {+ t.Fatalf("write CA certificate: %v", err)+ }++ connector, err := NewWebDAVConnector(map[string]any{+ "base_url": server.URL,+ "ca_cert_path": caCertPath,+ })+ if err != nil {+ t.Fatalf("NewWebDAVConnector failed: %v", err)+ }+ ctx, cancel := context.WithTimeout(context.Background(), webdavRequestTimeout)+ defer cancel()+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, server.URL, nil)+ if err != nil {+ t.Fatalf("create request: %v", err)+ }+ resp, err := connector.client.httpClient.Do(req)+ if err != nil {+ t.Fatalf("request with custom CA failed: %v", err)+ }+ defer resp.Body.Close()+}++func TestNewWebDAVConnectorRejectsInvalidCACertificate(t *testing.T) {+ tests := []struct {+ name string+ caCertPath func(t *testing.T) string+ }{+ {+ name: "missing file",+ caCertPath: func(t *testing.T) string {+ return filepath.Join(t.TempDir(), "missing.pem")+ },+ },+ {+ name: "invalid PEM",+ caCertPath: func(t *testing.T) string {+ path := filepath.Join(t.TempDir(), "invalid.pem")+ if err := os.WriteFile(path, []byte("not a certificate"), 0o600); err != nil {+ t.Fatalf("write invalid CA certificate: %v", err)+ }+ return path+ },+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ _, err := NewWebDAVConnector(map[string]any{"ca_cert_path": tt.caCertPath(t)})+ var validationErr *ConnectorValidationError+ if !errors.As(err, &validationErr) {+ t.Fatalf("error = %v, want ConnectorValidationError", err)+ }+ })+ }+}+func TestWebDAVConnectorValidate(t *testing.T) {server, _, _ := newWebDAVTestServer(t, webDAVTestTree())diff --git a/rag/svr/sync_data_source.py b/rag/svr/sync_data_source.pyindex 1245ecb9dd..e17d0051d2 100644--- a/rag/svr/sync_data_source.py+++ b/rag/svr/sync_data_source.py@@ -1385,6 +1385,7 @@ class WebDAV(SyncBase):base_url=self.conf["base_url"],remote_path=self.conf.get("remote_path", "/"),batch_size=batch_size,+ ca_cert_path=self.conf.get("ca_cert_path"),)self.connector.set_allow_images(self.conf.get("allow_images", False))self.connector.load_credentials(self.conf["credentials"])diff --git a/test/unit_test/data_source/test_webdav_connector_unit.py b/test/unit_test/data_source/test_webdav_connector_unit.pyindex 786f1820e2..545b3b80eb 100644--- a/test/unit_test/data_source/test_webdav_connector_unit.py+++ b/test/unit_test/data_source/test_webdav_connector_unit.py@@ -21,6 +21,7 @@ from datetime import datetime, timezonefrom enum import IntFlag, autofrom pathlib import Pathfrom types import ModuleType+from unittest.mock import Mockimport pytest@@ -149,6 +150,70 @@ webdav_connector = _load_webdav_connector_module()WebDAVConnector = webdav_connector.WebDAVConnector+def test_load_credentials_preserves_default_ssl_verification(monkeypatch):+ """Preserve the WebDAV client's default SSL verification without a custom CA."""+ webdav_client = Mock(return_value=object())+ monkeypatch.setattr(webdav_connector, "WebDAVClient", webdav_client)++ connector = WebDAVConnector("https://webdav.example")+ connector.load_credentials({"username": "user", "password": "password"})++ webdav_client.assert_called_once_with(+ base_url="https://webdav.example",+ auth=("user", "password"),+ )+++def test_build_connector_passes_custom_ca_certificate_path(monkeypatch):+ """Pass a configured custom CA certificate path through the factory."""+ webdav_client = Mock(return_value=object())+ monkeypatch.setattr(webdav_connector, "WebDAVClient", webdav_client)++ connector = WebDAVConnector.build_connector(+ {+ "base_url": "https://webdav.example",+ "ca_cert_path": "/etc/ssl/certs/webdav-ca.pem",+ "credentials": {"username": "user", "password": "password"},+ }+ )++ assert connector.ca_cert_path == "/etc/ssl/certs/webdav-ca.pem"+ webdav_client.assert_called_once_with(+ base_url="https://webdav.example",+ auth=("user", "password"),+ verify="/etc/ssl/certs/webdav-ca.pem",+ )+++def test_build_connector_rejects_non_string_ca_certificate_path():+ """Reject dynamically typed custom CA values before path normalization."""+ with pytest.raises(webdav_connector.ConnectorValidationError, match="CA certificate path must be a string"):+ WebDAVConnector.build_connector(+ {+ "base_url": "https://webdav.example",+ "ca_cert_path": 123,+ }+ )+++def test_load_credentials_reports_invalid_ca_as_validation_error(monkeypatch):+ """Report custom CA loading failures as configuration errors."""+ webdav_client = Mock(side_effect=OSError("invalid CA certificate"))+ monkeypatch.setattr(webdav_connector, "WebDAVClient", webdav_client)++ connector = WebDAVConnector(+ "https://webdav.example",+ ca_cert_path="/etc/ssl/certs/invalid.pem",+ )++ with pytest.raises(webdav_connector.ConnectorValidationError, match="Failed to configure WebDAV TLS verification"):+ connector.load_credentials({"username": "user", "password": "password"})++class _FakeClient:def __init__(self):self.downloaded_paths = []diff --git a/web/src/locales/en.ts b/web/src/locales/en.tsindex 553eee93b1..8ef30c6b50 100644--- a/web/src/locales/en.ts+++ b/web/src/locales/en.ts@@ -1564,6 +1564,8 @@ Example: Virtual Hosted Style`,webdavDescription: 'Connect to WebDAV servers to sync files.',webdavRemotePathTip:'Optional: Specify a folder path on the WebDAV server (e.g., /Documents). Leave empty to sync from root.',+ webdavCaCertPathTip:+ 'Optional: Path to a CA certificate bundle mounted inside the RAGFlow container.',google_driveTokenTip:'Upload the OAuth token JSON generated from the OAuth helper or Google Cloud Console. You may also upload a client_secret JSON from an "installed" or "web" application. If this is your first sync, a browser window will open to complete the OAuth consent. If the JSON already contains a refresh token, it will be reused automatically.',google_drivePrimaryAdminTip:diff --git a/web/src/pages/user-setting/data-source/constant/index.tsx b/web/src/pages/user-setting/data-source/constant/index.tsxindex 4932a3263f..977a69aabb 100644--- a/web/src/pages/user-setting/data-source/constant/index.tsx+++ b/web/src/pages/user-setting/data-source/constant/index.tsx@@ -1199,6 +1199,14 @@ const generateDataSourceFormFields = (t: TFunction) => ({placeholder: '/',tooltip: t('setting.webdavRemotePathTip'),},+ {+ label: 'Custom CA Certificate Path',+ name: 'config.ca_cert_path',+ type: FormFieldType.Text,+ required: false,+ placeholder: '/etc/ssl/certs/webdav-ca.pem',+ tooltip: t('setting.webdavCaCertPathTip'),+ },],[DataSourceKey.DROPBOX]: [{