Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat[syncer]: add connection test with unified connector validation (#18281)

Haruko386602f55deeddf
diff --git a/internal/handler/connector.go b/internal/handler/connector.go
index a8ab41c8e5..dcb32f46b1 100644
--- a/internal/handler/connector.go
+++ b/internal/handler/connector.go
@@ -20,6 +20,7 @@ import (
"context"
"encoding/json"
"errors"
+ "io"
"net/http"
"strconv"
"strings"
@@ -41,7 +42,7 @@ type connectorServiceIface interface {
DeleteConnector(ctx context.Context, connectorID, userID string) (bool, common.ErrorCode, error)
RebuildConnector(ctx context.Context, connectorID, userID, kbID string) (bool, common.ErrorCode, error)
ResumeFailedSync(ctx context.Context, connectorID, userID string, req *service.ResumeFailedSyncRequest) (bool, common.ErrorCode, error)
- TestConnector(ctx context.Context, connectorID, userID string) error
+ TestConnector(ctx context.Context, connectorID, userID string, config entity.JSONMap) error
UpdateConnector(ctx context.Context, connectorID, userID string, req *service.UpdateConnectorRequest) (*entity.Connector, common.ErrorCode, error)
StartGoogleWebOAuth(ctx context.Context, userID, source string, req *service.StartGoogleWebOAuthRequest) (*service.StartGoogleWebOAuthResponse, common.ErrorCode, error)
GoogleWebOAuthCallback(ctx context.Context, source, stateID, oauthError, errorDescription, code string) string
@@ -334,7 +335,12 @@ func (h *ConnectorHandler) CreateConnector(c *gin.Context) {
common.SuccessWithData(c, connector, "success")
}
-// TestConnector validates an accessible connector's stored credentials.
+type testConnectorRequest struct {
+ Source string `json:"source"`
+ Config entity.JSONMap `json:"config"`
+}
+
+// TestConnector validates connector settings.
// @Summary Test Connector
// @Description Validate connector credentials / connection (equivalent to Python's test_connector)
// @Tags connector
@@ -342,6 +348,7 @@ func (h *ConnectorHandler) CreateConnector(c *gin.Context) {
// @Param connector_id path string true "connector ID"
// @Router /api/v1/connectors/{connector_id}/test [post]
func (h *ConnectorHandler) TestConnector(c *gin.Context) {
+ // check user and connector
user, errorCode, errorMessage := GetUser(c)
if errorCode != common.CodeSuccess {
common.ErrorWithCode(c, errorCode, errorMessage)
@@ -356,7 +363,24 @@ func (h *ConnectorHandler) TestConnector(c *gin.Context) {
ctx := c.Request.Context()
- err := h.connectorService.TestConnector(ctx, connectorID, user.ID)
+ // build request
+ var request entity.JSONMap
+ if c.Request.Body != nil && c.Request.ContentLength != 0 {
+ var body testConnectorRequest
+ if err := c.ShouldBindJSON(&body); err != nil && !errors.Is(err, io.EOF) {
+ common.ResponseWithHttpCodeData(c, http.StatusBadRequest, common.CodeBadRequest, nil, err.Error())
+ return
+ }
+ // get source and config from web
+ if body.Source != "" || body.Config != nil {
+ request = entity.JSONMap{
+ "source": body.Source,
+ "config": body.Config,
+ }
+ }
+ }
+
+ err := h.connectorService.TestConnector(ctx, connectorID, user.ID, request)
if errors.Is(err, service.ErrConnectorTestUnsupported) {
connectorErrorResponse(c, err)
return
diff --git a/internal/handler/connector_test.go b/internal/handler/connector_test.go
index 3b2d744a44..bd137f706c 100644
--- a/internal/handler/connector_test.go
+++ b/internal/handler/connector_test.go
@@ -37,7 +37,7 @@ func (s fakeConnectorService) ListConnectors(context.Context, string) (*service.
return &service.ListConnectorsResponse{}, nil
}
-func (s fakeConnectorService) TestConnector(context.Context, string, string) error {
+func (s fakeConnectorService) TestConnector(context.Context, string, string, entity.JSONMap) error {
return s.err
}
diff --git a/internal/service/connector.go b/internal/service/connector.go
index 1458a73d84..ec77827170 100644
--- a/internal/service/connector.go
+++ b/internal/service/connector.go
@@ -78,15 +78,15 @@ var (
ErrConnectorNotFound = errors.New("can't find this Connector")
// ErrConnectorNoAuth mirrors Python's "no authorization" denial.
ErrConnectorNoAuth = errors.New("no authorization")
- // ErrConnectorTestUnsupported is returned for connector sources whose
- // validation path is not yet ported to Go.
- ErrConnectorTestUnsupported = errors.New("test endpoint currently supports only REST API connectors")
+ // ErrConnectorTestUnsupported is returned for connector sources without a settings validator.
+ ErrConnectorTestUnsupported = errors.New("connector test is not supported for this source")
)
// ConnectorService connector service
type ConnectorService struct {
- connectorDAO *dao.ConnectorDAO
- userTenantDAO *dao.UserTenantDAO
+ connectorDAO *dao.ConnectorDAO
+ userTenantDAO *dao.UserTenantDAO
+ connectorRegistry *syncerconnector.Registry
}
type syncTaskPublisher interface {
@@ -123,11 +123,18 @@ var getSyncCheckpointDeleter = func() (syncCheckpointDeleter, bool) {
// NewConnectorService create connector service
func NewConnectorService() *ConnectorService {
return &ConnectorService{
- connectorDAO: dao.NewConnectorDAO(),
- userTenantDAO: dao.NewUserTenantDAO(),
+ connectorDAO: dao.NewConnectorDAO(),
+ userTenantDAO: dao.NewUserTenantDAO(),
+ connectorRegistry: newConnectorRegistry(),
}
}
+func newConnectorRegistry() *syncerconnector.Registry {
+ registry := syncerconnector.NewRegistry()
+ syncerconnector.RegisterBuiltIns(registry)
+ return registry
+}
+
// ListConnectorsResponse list connectors response
type ListConnectorsResponse struct {
Connectors []*dao.ConnectorListItem `json:"connectors"`
@@ -365,7 +372,6 @@ func (s *ConnectorService) ListConnectors(ctx context.Context, userID string) (*
}
// accessible reports whether the user can access the connector's tenant.
-// Mirrors Python's ConnectorService.accessible: owner access plus joined tenants.
func (s *ConnectorService) accessible(ctx context.Context, connectorID, userID string) (bool, error) {
conn, err := s.connectorDAO.GetByID(ctx, dao.DB, connectorID)
if err != nil {
@@ -388,43 +394,89 @@ func (s *ConnectorService) accessible(ctx context.Context, connectorID, userID s
return false, nil
}
-// TestConnector validates a connector's stored configuration.
-// Equivalent to Python's test_connector. Per-connector credential validation
-// lives in the Python common.data_source package and is not yet available in
-// Go; for now this verifies access, that the connector exists, that the source
-// is REST_API (the only source Python currently tests), and that credentials
-// are present in the stored config. It returns ErrConnectorTestUnsupported for
-// other sources.
-func (s *ConnectorService) TestConnector(ctx context.Context, connectorID, userID string) error {
- ok, err := s.accessible(ctx, connectorID, userID)
- if err != nil && errors.Is(err, ErrConnectorNotFound) {
- return ErrConnectorNotFound
+// TestConnector validates connector settings without persisting or syncing.
+func (s *ConnectorService) TestConnector(ctx context.Context, connectorID, userID string, config entity.JSONMap) error {
+ var storedConnector *entity.Connector
+ if connectorID != "" {
+ ok, err := s.accessible(ctx, connectorID, userID)
+ if err != nil && !errors.Is(err, ErrConnectorNotFound) {
+ return err
+ }
+ if err == nil && !ok {
+ return ErrConnectorNoAuth
+ }
+ if err == nil {
+ storedConnector, err = s.connectorDAO.GetByID(ctx, dao.DB, connectorID)
+ if err != nil {
+ return ErrConnectorNotFound
+ }
+ }
+ if errors.Is(err, ErrConnectorNotFound) && config == nil {
+ return ErrConnectorNotFound
+ }
}
+
+ source, connectorConfig, err := testConnectorSettings(storedConnector, config)
if err != nil {
return err
}
- if !ok {
- return ErrConnectorNoAuth
- }
-
- conn, err := s.connectorDAO.GetByID(ctx, dao.DB, connectorID)
+ connector, err := s.connectorRegistry.OpenFromConfig(source, connectorConfig)
if err != nil {
- return ErrConnectorNotFound
+ return err
}
-
- if conn.Source != "rest_api" {
+ validator, ok := connector.(syncerconnector.SettingValidator)
+ if !ok {
return ErrConnectorTestUnsupported
}
+ return validator.ValidateConnectorSetting(ctx, connectorConfig)
+}
- config := conn.Config
+func testConnectorSettings(stored *entity.Connector, request entity.JSONMap) (string, entity.JSONMap, error) {
+ source := ""
+ var config entity.JSONMap
+ if stored != nil {
+ source = strings.TrimSpace(stored.Source)
+ config = stored.Config
+ }
+ if request != nil {
+ if value := strings.TrimSpace(stringConfigValue(request["source"])); value != "" {
+ source = value
+ }
+ if nested, ok := request["config"]; ok {
+ config = jsonMapValue(nested)
+ } else if _, ok := request["source"]; !ok {
+ config = request
+ }
+ }
+ if source == "" {
+ return "", nil, fmt.Errorf("connector source is required")
+ }
if config == nil {
- return fmt.Errorf("connector configuration is missing")
+ return "", nil, fmt.Errorf("connector configuration is missing")
}
- creds, ok := config["credentials"].(map[string]interface{})
- if !ok || len(creds) == 0 {
- return fmt.Errorf("connector credentials are missing")
+ return source, config, nil
+}
+
+func jsonMapValue(value any) entity.JSONMap {
+ switch typed := value.(type) {
+ case entity.JSONMap:
+ return typed
+ case map[string]any:
+ return entity.JSONMap(typed)
+ default:
+ return nil
+ }
+}
+
+func stringConfigValue(value any) string {
+ switch typed := value.(type) {
+ case string:
+ return typed
+ case nil:
+ return ""
+ default:
+ return fmt.Sprint(typed)
}
- return nil
}
func (s *ConnectorService) StartGoogleWebOAuth(ctx context.Context, userID, source string, req *StartGoogleWebOAuthRequest) (*StartGoogleWebOAuthResponse, common.ErrorCode, error) {
diff --git a/internal/service/connector_test_connection_test.go b/internal/service/connector_test_connection_test.go
new file mode 100644
index 0000000000..925bcbbe09
--- /dev/null
+++ b/internal/service/connector_test_connection_test.go
@@ -0,0 +1,165 @@
+package service
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "testing"
+
+ "ragflow/internal/entity"
+ syncerconnector "ragflow/internal/syncer/connector"
+ connectormock "ragflow/internal/syncer/connector/mock"
+)
+
+func TestConnectorServiceTestConnectorUsesRequestConfig(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.Connector{}, &entity.UserTenant{}); err != nil {
+ t.Fatalf("migrate connector tables: %v", err)
+ }
+ if err := db.Create(&entity.Connector{
+ ID: "conn-1",
+ TenantID: "tenant-1",
+ Name: "conn-1",
+ Source: "mock",
+ InputType: "poll",
+ Config: entity.JSONMap{"from": "stored"},
+ Status: "0",
+ }).Error; err != nil {
+ t.Fatalf("insert connector: %v", err)
+ }
+
+ var capturedConfig map[string]any
+ registry := syncerconnector.NewRegistry()
+ registry.RegisterConfigFactory("mock", func(config map[string]any) (syncerconnector.Connector, error) {
+ capturedConfig = config
+ return &connectormock.Connector{}, nil
+ })
+ svc := NewConnectorService()
+ svc.connectorRegistry = registry
+
+ request := entity.JSONMap{
+ "source": "mock",
+ "config": entity.JSONMap{"from": "request"},
+ }
+ if err := svc.TestConnector(context.Background(), "conn-1", "tenant-1", request); err != nil {
+ t.Fatalf("TestConnector failed: %v", err)
+ }
+ if capturedConfig["from"] != "request" {
+ t.Fatalf("config = %v, want request config", capturedConfig)
+ }
+}
+
+func TestConnectorServiceTestConnectorAllowsUnsavedConnectorWithSource(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.Connector{}, &entity.UserTenant{}); err != nil {
+ t.Fatalf("migrate connector tables: %v", err)
+ }
+
+ registry := syncerconnector.NewRegistry()
+ registry.RegisterConfigFactory("mock", func(config map[string]any) (syncerconnector.Connector, error) {
+ return &connectormock.Connector{}, nil
+ })
+ svc := NewConnectorService()
+ svc.connectorRegistry = registry
+
+ err := svc.TestConnector(context.Background(), "missing", "tenant-1", entity.JSONMap{
+ "source": "mock",
+ "config": entity.JSONMap{"ok": true},
+ })
+ if err != nil {
+ t.Fatalf("TestConnector failed: %v", err)
+ }
+}
+
+func TestConnectorServiceTestConnectorRejectsMissingConfigForMissingConnector(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.Connector{}, &entity.UserTenant{}); err != nil {
+ t.Fatalf("migrate connector tables: %v", err)
+ }
+
+ err := NewConnectorService().TestConnector(context.Background(), "missing", "tenant-1", nil)
+ if !errors.Is(err, ErrConnectorNotFound) {
+ t.Fatalf("error = %v, want ErrConnectorNotFound", err)
+ }
+}
+
+func TestConnectorServiceTestConnectorRejectsUnauthorizedConnector(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.Connector{}, &entity.UserTenant{}); err != nil {
+ t.Fatalf("migrate connector tables: %v", err)
+ }
+ if err := db.Create(&entity.Connector{
+ ID: "conn-1",
+ TenantID: "tenant-1",
+ Name: "conn-1",
+ Source: "mock",
+ InputType: "poll",
+ Config: entity.JSONMap{"ok": true},
+ Status: "0",
+ }).Error; err != nil {
+ t.Fatalf("insert connector: %v", err)
+ }
+
+ err := NewConnectorService().TestConnector(context.Background(), "conn-1", "user-2", entity.JSONMap{
+ "source": "mock",
+ "config": entity.JSONMap{"ok": true},
+ })
+ if !errors.Is(err, ErrConnectorNoAuth) {
+ t.Fatalf("error = %v, want ErrConnectorNoAuth", err)
+ }
+}
+
+func TestConnectorServiceTestConnectorRejectsUnsupportedSource(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.Connector{}, &entity.UserTenant{}); err != nil {
+ t.Fatalf("migrate connector tables: %v", err)
+ }
+
+ err := NewConnectorService().TestConnector(context.Background(), "missing", "tenant-1", entity.JSONMap{
+ "source": "unknown",
+ "config": entity.JSONMap{"ok": true},
+ })
+ if err == nil || !strings.Contains(err.Error(), `unsupported connector source "unknown"`) {
+ t.Fatalf("error = %v, want unsupported source", err)
+ }
+}
+
+func TestConnectorServiceTestConnectorRejectsConnectorWithoutValidator(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ if err := db.AutoMigrate(&entity.Connector{}, &entity.UserTenant{}); err != nil {
+ t.Fatalf("migrate connector tables: %v", err)
+ }
+
+ registry := syncerconnector.NewRegistry()
+ registry.RegisterConfigFactory("plain", func(config map[string]any) (syncerconnector.Connector, error) {
+ return plainConnector{}, nil
+ })
+ svc := NewConnectorService()
+ svc.connectorRegistry = registry
+
+ err := svc.TestConnector(context.Background(), "missing", "tenant-1", entity.JSONMap{
+ "source": "plain",
+ "config": entity.JSONMap{"ok": true},
+ })
+ if !errors.Is(err, ErrConnectorTestUnsupported) {
+ t.Fatalf("error = %v, want ErrConnectorTestUnsupported", err)
+ }
+}
+
+type plainConnector struct{}
+
+func (plainConnector) Validate(context.Context) error { return nil }
+
+func (plainConnector) OpenSync(context.Context, syncerconnector.SyncRequest) (syncerconnector.SyncSession, error) {
+ return nil, nil
+}
+
+func (plainConnector) OpenPrune(context.Context, syncerconnector.PruneRequest) (syncerconnector.PruneSession, error) {
+ return nil, nil
+}
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
new file mode 100644
index 0000000000..cd2ef1b1fa
--- /dev/null
+++ b/internal/syncer/connector/builtin.go
@@ -0,0 +1,53 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "fmt"
+
+ "ragflow/internal/dao"
+)
+
+// RegisterBuiltIns wires every connector shipped in the server binary. Each
+// source registers both the task-context factory (used by the syncer runtime)
+// and the raw-config factory (used by the test-connection endpoint).
+func RegisterBuiltIns(registry *Registry) {
+ registerBuiltIn(registry, "rss", NewRSSConnector)
+ registerBuiltIn(registry, "github", NewGitHubConnector)
+ registerBuiltIn(registry, "gmail", NewGmailConnector)
+ registerBuiltIn(registry, "google-drive", NewGoogleDriveConnector)
+ registerBuiltIn(registry, "google_drive", NewGoogleDriveConnector)
+ registerBuiltIn(registry, "outlook", NewOutlookConnector)
+ registerBuiltIn(registry, "rest_api", NewRestAPIConnector)
+ registerBuiltIn(registry, "mysql", NewMySQLConnector)
+ registerBuiltIn(registry, "postgresql", NewPostgreSQLConnector)
+ registerBuiltIn(registry, "discord", NewDiscordConnector)
+}
+
+func registerBuiltIn[T Connector](registry *Registry, source string, factory func(map[string]any) (T, error)) {
+ registry.RegisterConfigFactory(source, func(config map[string]any) (Connector, error) {
+ return factory(config)
+ })
+ registry.Register(source, func(ctx context.Context, taskContext any) (Connector, error) {
+ row, ok := taskContext.(dao.SyncTaskContext)
+ if !ok {
+ return nil, fmt.Errorf("%s connector received an invalid task context", source)
+ }
+ return factory(map[string]any(row.Connector.Config))
+ })
+}
diff --git a/internal/syncer/connector/discord.go b/internal/syncer/connector/discord.go
index 800617a540..15448621e2 100644
--- a/internal/syncer/connector/discord.go
+++ b/internal/syncer/connector/discord.go
@@ -250,6 +250,23 @@ func (c *DiscordConnector) Validate(ctx context.Context) error {
return nil
}
+// ValidateConnectorSetting validates Discord settings from an unsaved config.
+func (c *DiscordConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ if err := c.Validate(ctx); err != nil {
+ return err
+ }
+ targets, err := c.listTargets(ctx)
+ if err != nil {
+ return err
+ }
+ if len(targets) == 0 {
+ return &ConnectorValidationError{Message: "Discord connector found no accessible text channels"}
+ }
+ return nil
+}
+
// OpenSync opens one sync session over the configured channels and threads.
func (c *DiscordConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
targets, err := c.listTargets(ctx)
diff --git a/internal/syncer/connector/discord_test.go b/internal/syncer/connector/discord_test.go
index fb18842fa6..9c17750ba5 100644
--- a/internal/syncer/connector/discord_test.go
+++ b/internal/syncer/connector/discord_test.go
@@ -275,6 +275,36 @@ func TestDiscordValidateNoNetwork(t *testing.T) {
}
}
+func TestDiscordConnectorValidateConnectorSetting(t *testing.T) {
+ fixture := newDiscordFixture()
+ server := httptest.NewServer(fixture.handler())
+ defer server.Close()
+
+ connector := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "server_ids": []any{"1001"},
+ "channels": []any{"general"},
+ })
+ connector.baseURL = server.URL
+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+ if fixture.requests.Load() == 0 {
+ t.Fatalf("ValidateConnectorSetting made no network requests")
+ }
+
+ empty := newDiscordTestConnector(t, map[string]any{
+ "credentials": map[string]any{"discord_bot_token": "token"},
+ "server_ids": []any{"1001"},
+ "channels": []any{"missing"},
+ })
+ empty.baseURL = server.URL
+ var valErr *ConnectorValidationError
+ if err := empty.ValidateConnectorSetting(context.Background(), nil); !errors.As(err, &valErr) {
+ t.Fatalf("ValidateConnectorSetting empty err = %v, want ConnectorValidationError", err)
+ }
+}
+
func TestDiscordOpenSyncMergesBatches(t *testing.T) {
fixture := newDiscordFixture()
base := mustTime(t, "2026-08-14T10:00:00Z")
diff --git a/internal/syncer/connector/github.go b/internal/syncer/connector/github.go
index 323f38c385..80d8f45fd1 100644
--- a/internal/syncer/connector/github.go
+++ b/internal/syncer/connector/github.go
@@ -75,7 +75,7 @@ func (c *GitHubConnector) Validate(ctx context.Context) error {
return fmt.Errorf("github connector is nil")
}
if c.owner == "" {
- return fmt.Errorf("Invalid connector settings: 'repo_owner' must be provided")
+ return fmt.Errorf("Invalid connector settings: 'repository_owner' must be provided")
}
if c.token == "" {
return fmt.Errorf("Missing github_access_token in credentials")
@@ -93,6 +93,29 @@ func (c *GitHubConnector) Validate(ctx context.Context) error {
return nil
}
+// ValidateConnectorSetting validates GitHub settings from an unsaved config.
+func (c *GitHubConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ if c == nil {
+ return fmt.Errorf("github connector is nil")
+ }
+ if c.owner == "" {
+ return fmt.Errorf("Invalid connector settings: 'repo_owner' must be provided")
+ }
+ if c.token == "" {
+ return fmt.Errorf("Missing github_access_token in credentials")
+ }
+ if c.batchSize <= 0 {
+ return fmt.Errorf("batch_size must be a positive integer")
+ }
+ var user map[string]any
+ if _, err := c.getJSON(ctx, c.apiURL("/user", nil), &user); err != nil {
+ return err
+ }
+ return nil
+}
+
// OpenSync opens one GitHub sync session.
func (c *GitHubConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
repos, err := c.listRepos(ctx)
diff --git a/internal/syncer/connector/gmail.go b/internal/syncer/connector/gmail.go
index 1380c5ffcd..39cd2e3489 100644
--- a/internal/syncer/connector/gmail.go
+++ b/internal/syncer/connector/gmail.go
@@ -90,6 +90,13 @@ func (c *GmailConnector) Validate(ctx context.Context) error {
return err
}
+// ValidateConnectorSetting validates Gmail settings from an unsaved config.
+func (c *GmailConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ return c.Validate(ctx)
+}
+
// OpenSync opens one Gmail sync session.
func (c *GmailConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
users, err := c.getUserEmails(ctx)
diff --git a/internal/syncer/connector/google_drive.go b/internal/syncer/connector/google_drive.go
index 9f3c712ad4..3a9af58dca 100644
--- a/internal/syncer/connector/google_drive.go
+++ b/internal/syncer/connector/google_drive.go
@@ -138,6 +138,21 @@ func (c *GoogleDriveConnector) Validate(ctx context.Context) error {
return err
}
+// ValidateConnectorSetting validates Google Drive settings from an unsaved config.
+func (c *GoogleDriveConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ if err := c.Validate(ctx); err != nil {
+ return err
+ }
+ client, err := c.clientForUser(ctx, c.primaryAdminEmail)
+ if err != nil {
+ return err
+ }
+ var about map[string]any
+ return c.getJSON(ctx, client, "https://www.googleapis.com/drive/v3/about?fields=user", &about)
+}
+
// OpenSync opens one Google Drive sync session.
func (c *GoogleDriveConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
scopes, err := c.buildScopes(ctx)
diff --git a/internal/syncer/connector/interface.go b/internal/syncer/connector/interface.go
index c63f145f1b..7092d0711d 100644
--- a/internal/syncer/connector/interface.go
+++ b/internal/syncer/connector/interface.go
@@ -16,7 +16,12 @@
package connector
-import "context"
+import (
+ "context"
+ "time"
+)
+
+const connectorSettingValidationTimeout = 7 * time.Second
// SyncSession streams source documents for one fixed sync window.
type SyncSession interface {
@@ -32,6 +37,11 @@ type Fetcher interface {
Fetch(ctx context.Context, ref FetchReference) ([]byte, error)
}
+// SettingValidator validates connector settings from an unsaved config.
+type SettingValidator interface {
+ ValidateConnectorSetting(ctx context.Context, request map[string]any) error
+}
+
// PruneSession streams a complete slim source snapshot.
type PruneSession interface {
// NextBatch returns the next slim snapshot batch.
diff --git a/internal/syncer/connector/mock/mock.go b/internal/syncer/connector/mock/mock.go
index efe26780eb..425a8bdd2d 100644
--- a/internal/syncer/connector/mock/mock.go
+++ b/internal/syncer/connector/mock/mock.go
@@ -24,15 +24,16 @@ import (
// Connector is a programmable connector for syncer tests.
type Connector struct {
- ValidateErr error
- SyncBatches []syncerconnector.SyncBatch
- SyncErrAt int
- PruneBatches []syncerconnector.PruneBatch
- PruneErrAt int
- FetchBlobs map[string][]byte
- OnSyncBatch func(index int)
- OnPruneBatch func(index int)
- SyncRequests []syncerconnector.SyncRequest
+ ValidateErr error
+ ValidateConnectorSettingErr error
+ SyncBatches []syncerconnector.SyncBatch
+ SyncErrAt int
+ PruneBatches []syncerconnector.PruneBatch
+ PruneErrAt int
+ FetchBlobs map[string][]byte
+ OnSyncBatch func(index int)
+ OnPruneBatch func(index int)
+ SyncRequests []syncerconnector.SyncRequest
}
// Validate returns the configured validation error.
@@ -40,6 +41,11 @@ func (c *Connector) Validate(ctx context.Context) error {
return c.ValidateErr
}
+// ValidateConnectorSetting returns the configured settings validation error.
+func (c *Connector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ return c.ValidateConnectorSettingErr
+}
+
// OpenSync opens a mock sync session.
func (c *Connector) OpenSync(ctx context.Context, request syncerconnector.SyncRequest) (syncerconnector.SyncSession, error) {
c.SyncRequests = append(c.SyncRequests, request)
diff --git a/internal/syncer/connector/mysql.go b/internal/syncer/connector/mysql.go
index e9172e2fae..fb5eecaad3 100644
--- a/internal/syncer/connector/mysql.go
+++ b/internal/syncer/connector/mysql.go
@@ -100,13 +100,19 @@ func (c *MySQLConnector) Validate(ctx context.Context) error {
return fmt.Errorf("Failed to connect to MySQL: %w", err)
}
defer db.Close()
- var one int
- if err := db.QueryRowContext(ctx, "SELECT 1").Scan(&one); err != nil {
+ if err := db.PingContext(ctx); err != nil {
return fmt.Errorf("Failed to connect to MySQL: %w", err)
}
return nil
}
+// ValidateConnectorSetting validates MySQL settings from an unsaved config.
+func (c *MySQLConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ return c.Validate(ctx)
+}
+
// OpenSync opens one MySQL sync session.
func (c *MySQLConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
db, err := c.open()
diff --git a/internal/syncer/connector/mysql_test.go b/internal/syncer/connector/mysql_test.go
index c1c25dd30d..7cfd4d5ff3 100644
--- a/internal/syncer/connector/mysql_test.go
+++ b/internal/syncer/connector/mysql_test.go
@@ -48,7 +48,7 @@ func newFixtureMySQLConnector(t *testing.T, config map[string]any, expect func(m
if err != nil {
t.Fatalf("NewMySQLConnector failed: %v", err)
}
- db, mock, err := sqlmock.New()
+ db, mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true))
if err != nil {
t.Fatalf("sqlmock.New failed: %v", err)
}
@@ -288,12 +288,10 @@ func TestMySQLConnectorMD5FallbackID(t *testing.T) {
}
}
-// TestMySQLConnectorValidate verifies the SELECT 1 probe and failure paths.
+// TestMySQLConnectorValidate verifies the connection probe and failure paths.
func TestMySQLConnectorValidate(t *testing.T) {
connector := newFixtureMySQLConnector(t, nil, func(mock sqlmock.Sqlmock) {
- mock.ExpectQuery(regexp.QuoteMeta("SELECT 1")).WillReturnRows(
- sqlmock.NewRows([]string{"1"}).AddRow(1),
- )
+ mock.ExpectPing()
})
if err := connector.Validate(context.Background()); err != nil {
t.Fatalf("Validate failed: %v", err)
diff --git a/internal/syncer/connector/outlook.go b/internal/syncer/connector/outlook.go
index bcf1103e7d..456d77ef63 100644
--- a/internal/syncer/connector/outlook.go
+++ b/internal/syncer/connector/outlook.go
@@ -108,6 +108,13 @@ func (c *OutlookConnector) Validate(ctx context.Context) error {
return err
}
+// ValidateConnectorSetting validates Outlook settings from an unsaved config.
+func (c *OutlookConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ return c.Validate(ctx)
+}
+
func (c *OutlookConnector) effectiveBatchSize() int {
if c.batchSize > 0 {
return c.batchSize
diff --git a/internal/syncer/connector/outlook_test.go b/internal/syncer/connector/outlook_test.go
index c054c1ea71..e2e7c17906 100644
--- a/internal/syncer/connector/outlook_test.go
+++ b/internal/syncer/connector/outlook_test.go
@@ -236,6 +236,29 @@ func TestOutlookGetJSONRefreshesTokenAfterUnauthorized(t *testing.T) {
}
}
+func TestOutlookConnectorValidateConnectorSetting(t *testing.T) {
+ connector := newFixtureOutlookConnector()
+ var probed bool
+ connector.httpClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+ if req.URL.Path != "/v1.0/users/user-1" {
+ t.Fatalf("path = %q, want /v1.0/users/user-1", req.URL.Path)
+ }
+ probed = true
+ return &http.Response{
+ StatusCode: http.StatusOK,
+ Header: make(http.Header),
+ Body: io.NopCloser(strings.NewReader(`{"id":"user-1"}`)),
+ }, nil
+ })}
+
+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+ if !probed {
+ t.Fatalf("ValidateConnectorSetting did not probe Outlook user")
+ }
+}
+
type fixtureOutlookConnector struct {
*OutlookConnector
}
diff --git a/internal/syncer/connector/postgresql.go b/internal/syncer/connector/postgresql.go
index a0dd356a0a..c4bb70e70a 100644
--- a/internal/syncer/connector/postgresql.go
+++ b/internal/syncer/connector/postgresql.go
@@ -113,13 +113,19 @@ func (c *PostgreSQLConnector) Validate(ctx context.Context) error {
return fmt.Errorf("Failed to connect to PostgreSQL: %w", err)
}
defer db.Close()
- var one int
- if err := db.QueryRowContext(ctx, "SELECT 1").Scan(&one); err != nil {
+ if err := db.PingContext(ctx); err != nil {
return fmt.Errorf("Failed to connect to PostgreSQL: %w", err)
}
return nil
}
+// ValidateConnectorSetting validates PostgreSQL settings from an unsaved config.
+func (c *PostgreSQLConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ return c.Validate(ctx)
+}
+
// OpenSync opens one PostgreSQL sync session.
func (c *PostgreSQLConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
db, err := c.open()
diff --git a/internal/syncer/connector/postgresql_test.go b/internal/syncer/connector/postgresql_test.go
index 1996f91de2..764aa0d8cf 100644
--- a/internal/syncer/connector/postgresql_test.go
+++ b/internal/syncer/connector/postgresql_test.go
@@ -48,7 +48,7 @@ func newFixturePostgresConnector(t *testing.T, config map[string]any, expect fun
if err != nil {
t.Fatalf("NewPostgreSQLConnector failed: %v", err)
}
- db, mock, err := sqlmock.New()
+ db, mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true))
if err != nil {
t.Fatalf("sqlmock.New failed: %v", err)
}
@@ -319,9 +319,7 @@ func TestPostgreSQLConnectorOpenSyncMixedCaseTable(t *testing.T) {
// TestPostgreSQLConnectorValidate verifies the probe and dialect-specific error message.
func TestPostgreSQLConnectorValidate(t *testing.T) {
connector := newFixturePostgresConnector(t, nil, func(mock sqlmock.Sqlmock) {
- mock.ExpectQuery(regexp.QuoteMeta("SELECT 1")).WillReturnRows(
- sqlmock.NewRows([]string{"1"}).AddRow(1),
- )
+ mock.ExpectPing()
})
if err := connector.Validate(context.Background()); err != nil {
t.Fatalf("Validate failed: %v", err)
diff --git a/internal/syncer/connector/registry.go b/internal/syncer/connector/registry.go
index eb0db2287f..160649ae54 100644
--- a/internal/syncer/connector/registry.go
+++ b/internal/syncer/connector/registry.go
@@ -25,15 +25,22 @@ import (
// Factory creates a connector for a task context.
type Factory func(ctx context.Context, taskContext any) (Connector, error)
-// Registry connector source names to Go connector factories.
+// ConfigFactory creates a connector from raw connector config.
+type ConfigFactory func(config map[string]any) (Connector, error)
+
+// Registry maps connector source names to Go connector factories.
type Registry struct {
- mu sync.RWMutex
- factories map[string]Factory
+ mu sync.RWMutex
+ factories map[string]Factory
+ configFactories map[string]ConfigFactory
}
// NewRegistry creates an empty connector registry.
func NewRegistry() *Registry {
- return &Registry{factories: map[string]Factory{}}
+ return &Registry{
+ factories: map[string]Factory{},
+ configFactories: map[string]ConfigFactory{},
+ }
}
// Register adds or replaces a connector factory.
@@ -43,6 +50,13 @@ func (r *Registry) Register(source string, factory Factory) {
r.factories[source] = factory
}
+// RegisterConfigFactory adds or replaces a raw config connector factory.
+func (r *Registry) RegisterConfigFactory(source string, factory ConfigFactory) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.configFactories[source] = factory
+}
+
// Open creates a connector for a task context.
func (r *Registry) Open(ctx context.Context, taskContext any) (Connector, error) {
row, ok := taskContext.(interface{ ConnectorSource() string })
@@ -64,6 +78,17 @@ func (r *Registry) Open(ctx context.Context, taskContext any) (Connector, error)
return r.openSource(ctx, source, taskContext)
}
+// OpenFromConfig builds a connector from a raw config map.
+func (r *Registry) OpenFromConfig(source string, config map[string]any) (Connector, error) {
+ r.mu.RLock()
+ factory := r.configFactories[source]
+ r.mu.RUnlock()
+ if factory == nil {
+ return nil, fmt.Errorf("unsupported connector source %q", source)
+ }
+ return factory(config)
+}
+
// openSource creates a connector for a known source.
func (r *Registry) openSource(ctx context.Context, source string, taskContext any) (Connector, error) {
r.mu.RLock()
diff --git a/internal/syncer/connector/registry_test.go b/internal/syncer/connector/registry_test.go
new file mode 100644
index 0000000000..79662fa34a
--- /dev/null
+++ b/internal/syncer/connector/registry_test.go
@@ -0,0 +1,42 @@
+package connector
+
+import (
+ "context"
+ "strings"
+ "testing"
+)
+
+func TestRegistryOpenFromConfig(t *testing.T) {
+ registry := NewRegistry()
+ registry.RegisterConfigFactory("rss", func(config map[string]any) (Connector, error) {
+ return NewRSSConnector(config)
+ })
+
+ connector, err := registry.OpenFromConfig("rss", map[string]any{"feed_url": "https://example.com/feed.xml"})
+ if err != nil {
+ t.Fatalf("OpenFromConfig failed: %v", err)
+ }
+ if _, ok := connector.(*RSSConnector); !ok {
+ t.Fatalf("connector type = %T, want *RSSConnector", connector)
+ }
+
+ _, err = registry.OpenFromConfig("missing", map[string]any{})
+ if err == nil || !strings.Contains(err.Error(), `unsupported connector source "missing"`) {
+ t.Fatalf("unsupported source error = %v", err)
+ }
+}
+
+func TestRegistryOpenUsesTaskFactory(t *testing.T) {
+ registry := NewRegistry()
+ registry.Register("rss", func(ctx context.Context, taskContext any) (Connector, error) {
+ return NewRSSConnector(map[string]any{"feed_url": "https://example.com/feed.xml"})
+ })
+
+ connector, err := registry.Open(context.Background(), struct{ Connector struct{ Source string } }{Connector: struct{ Source string }{Source: "rss"}})
+ if err != nil {
+ t.Fatalf("Open failed: %v", err)
+ }
+ if _, ok := connector.(*RSSConnector); !ok {
+ t.Fatalf("connector type = %T, want *RSSConnector", connector)
+ }
+}
diff --git a/internal/syncer/connector/rest_api.go b/internal/syncer/connector/rest_api.go
index 9d08756340..936975238d 100644
--- a/internal/syncer/connector/rest_api.go
+++ b/internal/syncer/connector/rest_api.go
@@ -520,6 +520,13 @@ func (c *RestAPIConnector) ValidateLive(ctx context.Context) error {
return err
}
+// ValidateConnectorSetting validates REST API settings from an unsaved config.
+func (c *RestAPIConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ return c.ValidateLive(ctx)
+}
+
// OpenSync opens one sync session. Incremental runs filter documents to the
// request window (start <= updated_at < end).
func (c *RestAPIConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
diff --git a/internal/syncer/connector/rest_api_test.go b/internal/syncer/connector/rest_api_test.go
index b0777d8fe4..abfb8d72c4 100644
--- a/internal/syncer/connector/rest_api_test.go
+++ b/internal/syncer/connector/rest_api_test.go
@@ -751,6 +751,28 @@ func TestRestAPIValidateLive(t *testing.T) {
}
}
+func TestRestAPIValidateConnectorSetting(t *testing.T) {
+ withRestAPITestHooks(t)
+ var requests atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ requests.Add(1)
+ w.Header().Set("Content-Type", "application/json")
+ w.Write([]byte(`{"items": []}`))
+ }))
+ defer server.Close()
+
+ c := mustRestAPIConnector(t, map[string]any{
+ "url": server.URL,
+ "content_fields": "title",
+ })
+ if err := c.ValidateConnectorSetting(context.Background(), nil); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+ if requests.Load() != 1 {
+ t.Fatalf("requests = %d, want 1", requests.Load())
+ }
+}
+
func TestRestAPIMaxPages(t *testing.T) {
withRestAPITestHooks(t)
var requests atomic.Int32
diff --git a/internal/syncer/connector/rss.go b/internal/syncer/connector/rss.go
index f49672feff..44af351f67 100644
--- a/internal/syncer/connector/rss.go
+++ b/internal/syncer/connector/rss.go
@@ -135,6 +135,13 @@ func (c *RSSConnector) OpenPrune(ctx context.Context, request PruneRequest) (Pru
return &rssPruneSession{documents: documents, batchSize: c.batchSize}, nil
}
+// ValidateConnectorSetting validates RSS settings from an unsaved config.
+func (c *RSSConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ return c.Validate(ctx)
+}
+
// loadEntries fetches and parses the configured feed.
func (c *RSSConnector) loadEntries(ctx context.Context) ([]rssEntry, error) {
if c.entries != nil {
diff --git a/internal/syncer/connector/rss_test.go b/internal/syncer/connector/rss_test.go
index a293536e26..c10d793d02 100644
--- a/internal/syncer/connector/rss_test.go
+++ b/internal/syncer/connector/rss_test.go
@@ -137,6 +137,17 @@ func TestRSSConnectorOpenSyncResumesAfterCheckpoint(t *testing.T) {
}
}
+func TestRSSConnectorValidateConnectorSetting(t *testing.T) {
+ connector, err := NewRSSConnector(map[string]any{"feed_url": "https://example.com/feed.xml"})
+ if err != nil {
+ t.Fatalf("NewRSSConnector failed: %v", err)
+ }
+ connector.fetchFeed = staticRSSFeed
+ if err := connector.ValidateConnectorSetting(context.Background(), nil); err != nil {
+ t.Fatalf("ValidateConnectorSetting failed: %v", err)
+ }
+}
+
// staticRSSFeed returns a fixed RSS feed for tests.
func staticRSSFeed(ctx context.Context, feedURL string) ([]byte, error) {
return []byte(`<?xml version="1.0" encoding="UTF-8"?>
diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go
index 73ce5e0f5a..7f7d38e947 100644
--- a/internal/syncer/syncer.go
+++ b/internal/syncer/syncer.go
@@ -19,7 +19,6 @@ package syncer
import (
"context"
"errors"
- "fmt"
"sync"
"time"
@@ -57,7 +56,7 @@ func NewSyncer(taskWorkerCount int) *Syncer {
taskDAO := dao.NewSyncTaskDAO(nil)
registry := syncerconnector.NewRegistry()
- registerBuiltInConnectors(registry)
+ syncerconnector.RegisterBuiltIns(registry)
documentService := documentservice.NewDocumentService()
pruneService := service.NewSyncPruneService(documentService, nil)
@@ -167,27 +166,3 @@ func logSyncTaskDuration(taskContext service.SyncTaskContext, startedAt time.Tim
zap.Duration("elapsed", time.Since(startedAt)),
)
}
-
-// registerBuiltInConnectors registers datasource connectors available in the server binary.
-func registerBuiltInConnectors(registry *syncerconnector.Registry) {
- registerDAOConnector(registry, "rss", syncerconnector.NewRSSConnector)
- registerDAOConnector(registry, "github", syncerconnector.NewGitHubConnector)
- registerDAOConnector(registry, "gmail", syncerconnector.NewGmailConnector)
- registerDAOConnector(registry, "google-drive", syncerconnector.NewGoogleDriveConnector)
- registerDAOConnector(registry, "google_drive", syncerconnector.NewGoogleDriveConnector)
- registerDAOConnector(registry, "outlook", syncerconnector.NewOutlookConnector)
- registerDAOConnector(registry, "rest_api", syncerconnector.NewRestAPIConnector)
- registerDAOConnector(registry, "mysql", syncerconnector.NewMySQLConnector)
- registerDAOConnector(registry, "postgresql", syncerconnector.NewPostgreSQLConnector)
- registerDAOConnector(registry, "discord", syncerconnector.NewDiscordConnector)
-}
-
-func registerDAOConnector[T syncerconnector.Connector](registry *syncerconnector.Registry, source string, factory func(map[string]any) (T, error)) {
- registry.Register(source, func(ctx context.Context, taskContext any) (syncerconnector.Connector, error) {
- row, ok := taskContext.(dao.SyncTaskContext)
- if !ok {
- return nil, fmt.Errorf("%s connector received an invalid task context", source)
- }
- return factory(map[string]any(row.Connector.Config))
- })
-}
diff --git a/web/src/components/dynamic-form.tsx b/web/src/components/dynamic-form.tsx
index c42510bd85..61260f7c21 100644
--- a/web/src/components/dynamic-form.tsx
+++ b/web/src/components/dynamic-form.tsx
@@ -517,7 +517,6 @@ export const RenderField = ({
labelClassName={labelClassName || field.labelClassName}
>
{(fieldProps) => {
- console.log('multi select value', fieldProps);
const finalFieldProps = {
...fieldProps,
onValueChange: (value: string[]) => {
@@ -532,10 +531,6 @@ export const RenderField = ({
variant="inverted"
maxCount={100}
{...finalFieldProps}
- // onValueChange={(data) => {
- // console.log(data);
- // field.onChange?.(data);
- // }}
options={field.options as MultiSelectOptionType[]}
disabled={field.disabled}
/>
@@ -762,7 +757,6 @@ const DynamicForm = {
combinedErrors[key] = combinedErrors[key][0];
}
}
- console.log('combinedErrors', combinedErrors);
return {
values: Object.keys(combinedErrors).length ? {} : data,
errors: combinedErrors,
@@ -987,30 +981,17 @@ const DynamicForm = {
(async () => {
try {
const beValid = await form.trigger();
- console.log('form valid', beValid, form);
- // if (beValid) {
- // form.handleSubmit(async (values) => {
- // console.log('form values', values);
- // submitFunc?.(values);
- // })();
- // }
if (beValid && submitFunc) {
form.handleSubmit(async (values) => {
const filteredValues = (form as any).filterActiveValues
? (form as any).filterActiveValues(values)
: values;
- console.log(
- 'filtered form values in saving button',
- filteredValues,
- );
submitFunc(filteredValues);
})();
}
} catch (e) {
console.error(e);
- } finally {
- console.log('form submit3');
}
})();
}}
diff --git a/web/src/pages/user-setting/data-source/add-datasource-modal.tsx b/web/src/pages/user-setting/data-source/add-datasource-modal.tsx
index 7ba3536978..1d490ce229 100644
--- a/web/src/pages/user-setting/data-source/add-datasource-modal.tsx
+++ b/web/src/pages/user-setting/data-source/add-datasource-modal.tsx
@@ -22,7 +22,7 @@ import {
import { Button } from '@/components/ui/button';
import { Modal } from '@/components/ui/modal/modal';
import { IModalProps } from '@/interfaces/common';
-import { useMemo, useRef } from 'react';
+import { useCallback, useMemo, useRef } from 'react';
import { FieldValues } from 'react-hook-form';
import { useTranslation } from 'react-i18next';
import {
@@ -44,8 +44,10 @@ const AddDataSourceModal = ({
}: IModalProps<FieldValues> & { sourceData?: IDataSorceInfo }) => {
const { t } = useTranslation();
const formRef = useRef<DynamicFormRef>(null);
- const { loading: testLoading, handleTest } = useTestDataSource(formRef);
-
+ const { loading: testLoading, handleTest } = useTestDataSource(
+ formRef,
+ sourceData?.id,
+ );
const fields = useMemo<FormFieldConfig[]>(() => {
if (!sourceData) {
return [];
@@ -67,10 +69,28 @@ const AddDataSourceModal = ({
[sourceData],
);
- const handleOk = async (values?: FieldValues) => {
- await onOk?.(values);
+ const handleOk = useCallback(
+ async (values?: FieldValues) => {
+ const success = await onOk?.(values);
+ if (success) {
+ hideModal?.();
+ }
+ },
+ [hideModal, onOk],
+ );
+
+ const handleCancel = useCallback(() => {
hideModal?.();
- };
+ }, [hideModal]);
+
+ const handleSubmit = useCallback(
+ (values: FieldValues) => {
+ handleOk(values);
+ },
+ [handleOk],
+ );
+
+ const handleFormSubmit = useCallback(() => {}, []);
return (
<Modal
@@ -91,18 +111,12 @@ const AddDataSourceModal = ({
<DynamicForm.Root
ref={formRef}
fields={fields}
- onSubmit={(data) => {
- console.log(data);
- }}
+ onSubmit={handleFormSubmit}
defaultValues={defaultValues}
labelClassName="font-normal"
>
<div className=" absolute bottom-0 right-0 left-0 flex items-center justify-end w-full gap-2 py-6 px-6">
- <DynamicForm.CancelButton
- handleCancel={() => {
- hideModal?.();
- }}
- />
+ <DynamicForm.CancelButton handleCancel={handleCancel} />
<Button
type="button"
variant="outline"
@@ -115,9 +129,7 @@ const AddDataSourceModal = ({
<DynamicForm.SavingButton
submitLoading={loading || false}
buttonText={t('common.confirm')}
- submitFunc={(values: FieldValues) => {
- handleOk(values);
- }}
+ submitFunc={handleSubmit}
/>
</div>
</DynamicForm.Root>
diff --git a/web/src/pages/user-setting/data-source/hooks.ts b/web/src/pages/user-setting/data-source/hooks.ts
index fbad20f49e..ca69b0a26b 100644
--- a/web/src/pages/user-setting/data-source/hooks.ts
+++ b/web/src/pages/user-setting/data-source/hooks.ts
@@ -39,10 +39,21 @@ import {
IDataSourceLog,
} from './interface';
+const DataSourceKeys = {
+ list: () => ['data-source'] as const,
+ detail: (id?: string | null) => ['data-source-detail', id] as const,
+ logs: (
+ id: string | null,
+ pagination: ReturnType<typeof useGetPaginationWithRouter>['pagination'],
+ autoRefresh: boolean,
+ ) => ['data-source-logs', id, pagination, autoRefresh] as const,
+ logsPrefix: (id?: string | null) => ['data-source-logs', id] as const,
+};
+
export const useListDataSource = () => {
const { dataSourceInfo } = useDataSourceInfo();
const { data: list, isFetching } = useQuery<IDataSource[]>({
- queryKey: ['data-source'],
+ queryKey: DataSourceKeys.list(),
queryFn: async () => {
const { data } = await dataSourceService.dataSourceList();
return data.data;
@@ -83,9 +94,8 @@ export const useListDataSource = () => {
}
});
- console.log('🚀 ~ useListDataSource ~ sourceList:', sourceList);
return sourceList;
- }, [list]);
+ }, [dataSourceInfo, list]);
return { list, categorizedList: updatedDataSourceTemplates, isFetching };
};
@@ -112,28 +122,32 @@ export const useAddDataSource = ({ isEdit = false }: { isEdit?: boolean }) => {
const handleAddOk = useCallback(
async (data: any) => {
setAddLoading(true);
- const { data: res } = isEdit
- ? await dataSourceUpdate(data.id, {
- ...data,
- reschedule: true,
- })
- : await dataSourceService.dataSourceSet(data);
- console.log('🚀 ~ handleAddOk ~ code:', res.code);
- if (res.code === 0) {
- if (isEdit && res.data?.id) {
- queryClient.setQueryData(
- ['data-source-detail', res.data.id],
- res.data,
- );
- queryClient.invalidateQueries({
- queryKey: ['data-source-detail', res.data.id],
- });
+ try {
+ const { data: res } = isEdit
+ ? await dataSourceUpdate(data.id, {
+ ...data,
+ reschedule: true,
+ })
+ : await dataSourceService.dataSourceSet(data);
+ if (res.code === 0) {
+ if (isEdit && res.data?.id) {
+ queryClient.setQueryData(
+ DataSourceKeys.detail(res.data.id),
+ res.data,
+ );
+ queryClient.invalidateQueries({
+ queryKey: DataSourceKeys.detail(res.data.id),
+ });
+ }
+ queryClient.invalidateQueries({ queryKey: DataSourceKeys.list() });
+ message.success(t(`message.operated`));
+ hideAddingModal();
+ return true;
}
- queryClient.invalidateQueries({ queryKey: ['data-source'] });
- message.success(t(`message.operated`));
- hideAddingModal();
+ return false;
+ } finally {
+ setAddLoading(false);
}
- setAddLoading(false);
},
[hideAddingModal, isEdit, queryClient],
);
@@ -158,7 +172,7 @@ export const useLogListDataSource = (autoRefresh: boolean) => {
logs: IDataSourceLog[];
total: number;
}>({
- queryKey: ['data-source-logs', id, pagination, autoRefresh],
+ queryKey: DataSourceKeys.logs(id, pagination, autoRefresh),
refetchInterval: autoRefresh ? 15 * 1000 : false,
queryFn: async () => {
const { data } = await getDataSourceLogs(id as string, {
@@ -186,7 +200,7 @@ export const useDeleteDataSource = () => {
const { data } = await deleteDataSource(id);
if (data.code === 0) {
message.success(t(`message.deleted`));
- queryClient.invalidateQueries({ queryKey: ['data-source'] });
+ queryClient.invalidateQueries({ queryKey: DataSourceKeys.list() });
}
setDeleteLoading(false);
},
@@ -199,7 +213,7 @@ export const useFetchDataSourceDetail = () => {
const [currentQueryParameters] = useSearchParams();
const id = currentQueryParameters.get('id');
const { data } = useQuery<IDataSource>({
- queryKey: ['data-source-detail', id],
+ queryKey: DataSourceKeys.detail(id),
enabled: !!id,
queryFn: async () => {
const { data } = await featchDataSourceDetail(id as string);
@@ -228,7 +242,7 @@ export const useUpdateDataSourceStatus = () => {
});
if (data.code === 0) {
queryClient.setQueryData(
- ['data-source-detail', id],
+ DataSourceKeys.detail(id),
(previous?: IDataSource) => ({
...(previous || {}),
...(data.data || {}),
@@ -238,11 +252,11 @@ export const useUpdateDataSourceStatus = () => {
await Promise.all([
queryClient.invalidateQueries({
- queryKey: ['data-source-detail', id],
+ queryKey: DataSourceKeys.detail(id),
}),
- queryClient.invalidateQueries({ queryKey: ['data-source'] }),
+ queryClient.invalidateQueries({ queryKey: DataSourceKeys.list() }),
queryClient.invalidateQueries({
- queryKey: ['data-source-logs', id],
+ queryKey: DataSourceKeys.logsPrefix(id),
}),
]);
@@ -267,7 +281,6 @@ export const useDataSourceRebuild = () => {
kb_id: id as string,
});
if (data.code === 0) {
- // queryClient.invalidateQueries({ queryKey: ['data-source-detail', id] });
message.success(t(`message.operated`));
}
},
@@ -280,12 +293,18 @@ export const useTestDataSource = (
formRef: RefObject<DynamicFormRef | null>,
connectorId?: string,
) => {
+ const [currentQueryParameters] = useSearchParams();
+ const id = currentQueryParameters.get('id');
const [loading, setLoading] = useState(false);
const handleTest = useCallback(async () => {
const values = formRef.current?.getFilteredValues();
const source = values?.source;
- if (!source) return;
+ const connectorID = id || values?.id || connectorId || source;
+ if (!connectorID || !source) return;
+
+ const isValid = await formRef.current?.trigger();
+ if (!isValid) return;
setLoading(true);
try {
@@ -293,17 +312,21 @@ export const useTestDataSource = (
values?.config && typeof values.config === 'object'
? values.config
: {};
- const { data } = await testDataSource(connectorId || source, {
+ const { data } = await testDataSource(connectorID, {
source,
config,
});
if (data.code === 0) {
message.success(t('setting.dataSourceTestSuccess'));
+ } else {
+ message.error(data.message || t('setting.dataSourceTestFailed'));
}
+ } catch {
+ message.error(t('setting.dataSourceTestFailed'));
} finally {
setLoading(false);
}
- }, [formRef, connectorId]);
+ }, [connectorId, formRef, id]);
return { loading, handleTest };
};
diff --git a/web/src/pages/user-setting/data-source/index.tsx b/web/src/pages/user-setting/data-source/index.tsx
index 2bb137779f..9c14575758 100644
--- a/web/src/pages/user-setting/data-source/index.tsx
+++ b/web/src/pages/user-setting/data-source/index.tsx
@@ -153,8 +153,7 @@ const DataSource = () => {
loading={addLoading}
hideModal={hideAddingModal}
onOk={(data) => {
- console.log(data);
- handleAddOk(data);
+ return handleAddOk(data);
}}
sourceData={addSource}
/>