Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(syncer): add Feishu Wiki connector for the Go backend (#19453)

mkaaad3e1396d0422c
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index 68144a4beb..0dbeaba817 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -27,6 +27,7 @@ import (
// and the raw-config factory (used by the test-connection endpoint).
func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "confluence", NewConfluenceConnector)
+ registerBuiltIn(registry, "feishu_wiki", NewFeishuWikiConnector)
registerBuiltIn(registry, "rss", NewRSSConnector)
registerBuiltIn(registry, "sitemap", NewSitemapConnector)
registerBuiltIn(registry, "salesforce", NewSalesforceConnector)
diff --git a/internal/syncer/connector/feishu_wiki.go b/internal/syncer/connector/feishu_wiki.go
new file mode 100644
index 0000000000..c4583a1ee5
--- /dev/null
+++ b/internal/syncer/connector/feishu_wiki.go
@@ -0,0 +1,880 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "math/rand"
+ "net/http"
+ "net/url"
+ "path"
+ "sort"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+)
+
+const (
+ feishuOpenBaseURL = "https://open.feishu.cn"
+ feishuRequestTimeout = 60 * time.Second
+ defaultFeishuBatchSize = 2
+ maxFeishuBatchSize = 10
+ feishuNodePageSize = 50
+ defaultFeishuMaxFileSize = 50 * 1024 * 1024
+ feishuMaxResponseBody = 4 * 1024 * 1024
+)
+
+// feishuSupportedExtensions is the set of downloadable Wiki file extensions the
+// connector accepts, mirroring the Python connector's allowlist.
+var feishuSupportedExtensions = map[string]struct{}{
+ ".csv": {}, ".doc": {}, ".docx": {}, ".eml": {}, ".gif": {}, ".html": {},
+ ".jpeg": {}, ".jpg": {}, ".json": {}, ".md": {}, ".mdx": {}, ".pdf": {},
+ ".png": {}, ".ppt": {}, ".pptx": {}, ".tif": {}, ".txt": {}, ".xls": {},
+ ".xlsx": {},
+}
+
+// Retry/backoff knobs for Feishu API calls. They are package variables so tests
+// can shrink the delays (mirrors rest_api.go).
+var (
+ feishuRetryTries = 5
+ feishuRetryBaseDelay = time.Second
+ feishuRetryMaxDelay = 30 * time.Second
+ feishuRetryBackoff = 2.0
+ feishuRetryJitter = time.Second
+ feishu429MaxWaits = 5
+ feishu429DefaultWait = 5 * time.Second
+)
+
+// ErrFeishuWikiPruneUnsupported reports that the Feishu Wiki connector cannot
+// reconcile deletions; prune tasks then complete without deleting anything.
+var ErrFeishuWikiPruneUnsupported = fmt.Errorf("feishu wiki connector does not support prune: %w", ErrPruneUnsupported)
+
+// FeishuWikiConnector reads downloadable attachment files from one Feishu Wiki
+// subtree using an application's tenant credentials.
+type FeishuWikiConnector struct {
+ appID string
+ appSecret string
+ spaceID string
+ rootNodeToken string
+ includeExtensions map[string]struct{}
+ includeKeywords map[string]struct{}
+ excludeKeywords map[string]struct{}
+ batchSize int
+ maxFileSizeBytes int64
+ baseURL string
+ httpClient *http.Client
+
+ mu sync.Mutex
+ accessToken string
+ accessTokenExpiresAt time.Time
+}
+
+// NewFeishuWikiConnector creates a Feishu Wiki connector from Python-compatible
+// config. It performs synchronous config validation but no network I/O.
+func NewFeishuWikiConnector(config map[string]any) (*FeishuWikiConnector, error) {
+ credentials, _ := config["credentials"].(map[string]any)
+ batchSize, err := feishuConfigInt(config["batch_size"], defaultFeishuBatchSize)
+ if err != nil {
+ return nil, &ConnectorValidationError{Message: "Feishu Wiki batch_size must be an integer"}
+ }
+ maxFileSizeBytes, err := feishuConfigInt64(config["max_file_size_bytes"], defaultFeishuMaxFileSize)
+ if err != nil {
+ return nil, &ConnectorValidationError{Message: "Feishu Wiki max_file_size_bytes must be an integer"}
+ }
+ connector := &FeishuWikiConnector{
+ appID: strings.TrimSpace(stringConfig(credentials["app_id"])),
+ appSecret: strings.TrimSpace(stringConfig(credentials["app_secret"])),
+ spaceID: strings.TrimSpace(stringConfig(config["space_id"])),
+ rootNodeToken: strings.TrimSpace(stringConfig(config["root_node_token"])),
+ includeExtensions: feishuExtensionSet(config["include_extensions"]),
+ includeKeywords: feishuKeywordSet(config["include_keywords"]),
+ excludeKeywords: feishuKeywordSet(config["exclude_keywords"]),
+ batchSize: batchSize,
+ maxFileSizeBytes: maxFileSizeBytes,
+ baseURL: feishuOpenBaseURL,
+ httpClient: &http.Client{Timeout: feishuRequestTimeout},
+ }
+ if err := connector.validateConfig(); err != nil {
+ return nil, err
+ }
+ return connector, nil
+}
+
+// validateConfig validates configuration values without network I/O.
+func (c *FeishuWikiConnector) validateConfig() error {
+ if c.appID == "" || c.appSecret == "" {
+ return &ConnectorMissingCredentialError{Message: "Feishu Wiki"}
+ }
+ if c.spaceID == "" {
+ return &ConnectorValidationError{Message: "Feishu Wiki space ID is required"}
+ }
+ if c.rootNodeToken == "" {
+ return &ConnectorValidationError{Message: "Feishu Wiki root node token is required"}
+ }
+ if c.batchSize < 1 || c.batchSize > maxFeishuBatchSize {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Feishu Wiki batch_size must be between 1 and %d", maxFeishuBatchSize)}
+ }
+ if c.maxFileSizeBytes <= 0 {
+ return &ConnectorValidationError{Message: "Feishu Wiki max_file_size_bytes must be a positive integer"}
+ }
+ for ext := range c.includeExtensions {
+ if _, ok := feishuSupportedExtensions[ext]; !ok {
+ return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported Feishu Wiki file extension: %s", ext)}
+ }
+ }
+ return nil
+}
+
+// Validate validates configuration and credentials by authenticating and
+// listing the first page of the configured root node.
+func (c *FeishuWikiConnector) Validate(ctx context.Context) error {
+ if err := c.validateConfig(); err != nil {
+ return err
+ }
+ if _, err := c.getAccessToken(ctx); err != nil {
+ return err
+ }
+ if _, _, _, err := c.listChildPage(ctx, c.rootNodeToken, ""); err != nil {
+ return fmt.Errorf("Feishu Wiki validation failed: %w", err)
+ }
+ return nil
+}
+
+// ValidateConnectorSetting validates Feishu Wiki settings from an unsaved config.
+func (c *FeishuWikiConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ candidate, err := NewFeishuWikiConnector(request)
+ if err != nil {
+ return err
+ }
+ if c != nil {
+ if c.httpClient != nil {
+ candidate.httpClient = c.httpClient
+ }
+ if c.baseURL != "" {
+ candidate.baseURL = c.baseURL
+ }
+ }
+ return candidate.Validate(ctx)
+}
+
+// OpenSync opens one Feishu Wiki synchronization session.
+func (c *FeishuWikiConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ files, err := c.collectFiles(ctx)
+ if err != nil {
+ return nil, err
+ }
+ end := request.WindowEnd
+ if end.IsZero() {
+ end = time.Now().UTC()
+ }
+ accepted := make([]feishuWikiFile, 0, len(files))
+ for _, file := range files {
+ if request.WindowStart != nil || !request.WindowEnd.IsZero() {
+ var updatedAt *time.Time
+ if file.hasUpdatedAt {
+ updatedAt = &file.updatedAt
+ }
+ if !feishuWithinWindow(updatedAt, request.WindowStart, &end) {
+ continue
+ }
+ }
+ if !c.matchesFilters(file.title) {
+ continue
+ }
+ accepted = append(accepted, file)
+ }
+ sort.Slice(accepted, func(i, j int) bool {
+ return accepted[i].sourceID(c.spaceID) < accepted[j].sourceID(c.spaceID)
+ })
+ session := &feishuWikiSyncSession{
+ connector: c,
+ files: accepted,
+ batchSize: c.batchSize,
+ windowEnd: end,
+ }
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
+ return session, nil
+}
+
+// OpenPrune reports prune as unsupported: ordinary polling does not reconcile
+// source deletions (mirroring the Python connector).
+func (c *FeishuWikiConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ return nil, ErrFeishuWikiPruneUnsupported
+}
+
+// feishuWikiFile is the metadata for one downloadable Wiki file node.
+type feishuWikiFile struct {
+ nodeToken string
+ objToken string
+ title string
+ objType string
+ parentNodeToken string
+ updatedAt time.Time
+ hasUpdatedAt bool
+}
+
+func (f feishuWikiFile) sourceID(spaceID string) string {
+ return "feishu_wiki:" + spaceID + ":" + f.nodeToken
+}
+
+// feishuWikiSyncSession streams accepted Wiki files in fixed-size batches,
+// resuming from a committed SourceID anchor.
+type feishuWikiSyncSession struct {
+ connector *FeishuWikiConnector
+ files []feishuWikiFile
+ batchSize int
+ windowEnd time.Time
+ index int
+}
+
+// NextBatch returns the next source document batch.
+func (s *feishuWikiSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ if s.index >= len(s.files) {
+ return SyncBatch{}, io.EOF
+ }
+ documents := make([]SourceDocument, 0, s.batchSize)
+ var last feishuWikiFile
+ for attempts := 0; attempts < s.batchSize && s.index < len(s.files); attempts++ {
+ file := s.files[s.index]
+ s.index++
+ last = file
+ document, err := s.connector.buildDocument(ctx, file, s.windowEnd)
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ documents = append(documents, document)
+ }
+ return SyncBatch{Documents: documents, Checkpoint: feishuWikiCheckpoint(s.connector.spaceID, last)}, nil
+}
+
+// Close releases connector resources.
+func (s *feishuWikiSyncSession) Close() error {
+ return nil
+}
+
+// applyResume advances the session past the last committed SourceID anchor.
+func (s *feishuWikiSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ if sourceID == "" {
+ return fmt.Errorf("feishu wiki sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ for index, file := range s.files {
+ if file.sourceID(s.connector.spaceID) == sourceID {
+ s.index = index + 1
+ return nil
+ }
+ }
+ return fmt.Errorf("feishu wiki resume anchor %q was not found in the current listing: %w", sourceID, ErrSyncResumeInvalid)
+}
+
+func feishuWikiCheckpoint(spaceID string, file feishuWikiFile) *SyncCheckpoint {
+ checkpoint := &SyncCheckpoint{
+ Cursor: file.sourceID(spaceID),
+ SourceID: file.sourceID(spaceID),
+ }
+ if file.hasUpdatedAt {
+ updatedAt := file.updatedAt
+ checkpoint.UpdatedAt = &updatedAt
+ }
+ return checkpoint
+}
+
+// buildDocument downloads one file and maps it to a SourceDocument.
+func (c *FeishuWikiConnector) buildDocument(ctx context.Context, file feishuWikiFile, fallbackUpdatedAt time.Time) (SourceDocument, error) {
+ blob, err := c.downloadFile(ctx, file.objToken)
+ if err != nil {
+ return SourceDocument{}, err
+ }
+ updatedAt := fallbackUpdatedAt
+ if file.hasUpdatedAt {
+ updatedAt = file.updatedAt
+ }
+ parentNodeToken := file.parentNodeToken
+ if parentNodeToken == "" {
+ parentNodeToken = c.rootNodeToken
+ }
+ sha := sha256.Sum256(blob)
+ shaHex := hex.EncodeToString(sha[:])
+ return SourceDocument{
+ SourceID: file.sourceID(c.spaceID),
+ SemanticIdentifier: firstNonEmpty(file.title, file.nodeToken),
+ Extension: strings.ToLower(path.Ext(file.title)),
+ Blob: blob,
+ UpdatedAt: updatedAt,
+ SizeBytes: int64(len(blob)),
+ Fingerprint: shaHex[:32],
+ Metadata: map[string]any{
+ "source": "feishu_wiki",
+ "wiki_space_id": c.spaceID,
+ "wiki_node_token": file.nodeToken,
+ "wiki_object_token": file.objToken,
+ "wiki_object_type": file.objType,
+ "wiki_parent_node_token": parentNodeToken,
+ "wiki_url": fmt.Sprintf("https://feishu.cn/wiki/%s", file.nodeToken),
+ "source_updated_at": updatedAt.UTC().Format(time.RFC3339),
+ "content_sha256": shaHex,
+ },
+ }, nil
+}
+
+// matchesFilters applies the extension and keyword filters to a node title.
+func (c *FeishuWikiConnector) matchesFilters(title string) bool {
+ ext := strings.ToLower(path.Ext(title))
+ if _, ok := feishuSupportedExtensions[ext]; !ok {
+ return false
+ }
+ if len(c.includeExtensions) > 0 {
+ if _, ok := c.includeExtensions[ext]; !ok {
+ return false
+ }
+ }
+ normalizedTitle := strings.ToLower(title)
+ if len(c.includeKeywords) > 0 && !feishuContainsAny(normalizedTitle, c.includeKeywords) {
+ return false
+ }
+ return !feishuContainsAny(normalizedTitle, c.excludeKeywords)
+}
+
+func feishuContainsAny(value string, terms map[string]struct{}) bool {
+ for term := range terms {
+ if strings.Contains(value, term) {
+ return true
+ }
+ }
+ return false
+}
+
+// feishuWithinWindow reports whether a node update time falls in (start, end].
+// Nodes without a timestamp are always included so content is not silently
+// omitted.
+func feishuWithinWindow(updatedAt *time.Time, start, end *time.Time) bool {
+ if updatedAt == nil {
+ return true
+ }
+ if start != nil && !start.Before(*updatedAt) {
+ return false
+ }
+ return end == nil || !updatedAt.After(*end)
+}
+
+// collectFiles walks the configured Wiki subtree and returns all downloadable
+// file nodes (filters are applied later so the window can be applied at open).
+func (c *FeishuWikiConnector) collectFiles(ctx context.Context) ([]feishuWikiFile, error) {
+ var files []feishuWikiFile
+ stack := []string{c.rootNodeToken}
+ for len(stack) > 0 {
+ parent := stack[len(stack)-1]
+ stack = stack[:len(stack)-1]
+ pageToken := ""
+ for {
+ items, hasMore, next, err := c.listChildPage(ctx, parent, pageToken)
+ if err != nil {
+ return nil, err
+ }
+ for _, node := range items {
+ nodeToken := feishuString(node["node_token"])
+ objType := feishuString(node["obj_type"])
+ objToken := feishuString(node["obj_token"])
+ title := feishuString(node["title"])
+ hasChild, _ := node["has_child"].(bool)
+ if objType == "file" {
+ updatedAt, hasUpdatedAt := feishuParseTimestamp(node["obj_edit_time"])
+ files = append(files, feishuWikiFile{
+ nodeToken: nodeToken,
+ objToken: objToken,
+ title: title,
+ objType: objType,
+ parentNodeToken: feishuString(node["parent_node_token"]),
+ updatedAt: updatedAt,
+ hasUpdatedAt: hasUpdatedAt,
+ })
+ }
+ if hasChild && nodeToken != "" {
+ stack = append(stack, nodeToken)
+ }
+ }
+ if !hasMore {
+ break
+ }
+ if next == "" {
+ return nil, &ConnectorValidationError{Message: "Feishu Wiki pagination indicated more data without a page token"}
+ }
+ pageToken = next
+ }
+ }
+ return files, nil
+}
+
+// listChildPage lists one page of child nodes for a parent node.
+func (c *FeishuWikiConnector) listChildPage(ctx context.Context, parentNodeToken, pageToken string) ([]map[string]any, bool, string, error) {
+ token, err := c.getAccessToken(ctx)
+ if err != nil {
+ return nil, false, "", err
+ }
+ endpoint := fmt.Sprintf("%s/open-apis/wiki/v2/spaces/%s/nodes?page_size=%d", c.baseURL, url.PathEscape(c.spaceID), feishuNodePageSize)
+ if parentNodeToken != "" {
+ endpoint += "&parent_node_token=" + url.QueryEscape(parentNodeToken)
+ }
+ if pageToken != "" {
+ endpoint += "&page_token=" + url.QueryEscape(pageToken)
+ }
+ resp, err := c.feishuDoRequest(ctx, http.MethodGet, endpoint, nil, map[string]string{"Authorization": "Bearer " + token})
+ if err != nil {
+ return nil, false, "", err
+ }
+ payload, err := feishuDecodeResponse(resp, "list Wiki nodes")
+ if err != nil {
+ return nil, false, "", err
+ }
+ data, _ := payload["data"].(map[string]any)
+ if data == nil {
+ return nil, false, "", &ConnectorValidationError{Message: "Feishu Wiki node response contained invalid data"}
+ }
+ items := feishuNodeItems(data["items"])
+ hasMore, _ := data["has_more"].(bool)
+ return items, hasMore, feishuString(data["page_token"]), nil
+}
+
+func feishuNodeItems(value any) []map[string]any {
+ raw, ok := value.([]any)
+ if !ok {
+ return nil
+ }
+ items := make([]map[string]any, 0, len(raw))
+ for _, item := range raw {
+ if node, ok := item.(map[string]any); ok {
+ items = append(items, node)
+ }
+ }
+ return items
+}
+
+// getAccessToken returns a cached tenant access token, refreshing it when
+// expired or absent.
+func (c *FeishuWikiConnector) getAccessToken(ctx context.Context) (string, error) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if c.accessToken != "" && time.Now().Before(c.accessTokenExpiresAt) {
+ return c.accessToken, nil
+ }
+ body, err := json.Marshal(map[string]string{"app_id": c.appID, "app_secret": c.appSecret})
+ if err != nil {
+ return "", err
+ }
+ resp, err := c.feishuDoRequest(ctx, http.MethodPost, c.baseURL+"/open-apis/auth/v3/tenant_access_token/internal", body, nil)
+ if err != nil {
+ return "", err
+ }
+ payload, err := feishuDecodeResponse(resp, "authenticate")
+ if err != nil {
+ return "", err
+ }
+ accessToken := feishuString(payload["tenant_access_token"])
+ if accessToken == "" {
+ return "", &ConnectorValidationError{Message: "Feishu authentication response did not contain a tenant access token"}
+ }
+ expiresIn := feishuIntDefault(payload["expire"], 7200)
+ if expiresIn < 60 {
+ expiresIn = 7200
+ }
+ c.accessToken = accessToken
+ c.accessTokenExpiresAt = time.Now().Add(time.Duration(expiresIn-60) * time.Second)
+ return accessToken, nil
+}
+
+// downloadFile streams one Wiki file, enforcing the configured size limit and
+// rejecting Feishu JSON error envelopes so they are never ingested as content.
+func (c *FeishuWikiConnector) downloadFile(ctx context.Context, objectToken string) ([]byte, error) {
+ token, err := c.getAccessToken(ctx)
+ if err != nil {
+ return nil, err
+ }
+ endpoint := fmt.Sprintf("%s/open-apis/drive/v1/files/%s/download", c.baseURL, url.PathEscape(objectToken))
+ resp, err := c.feishuDoRequest(ctx, http.MethodGet, endpoint, nil, map[string]string{"Authorization": "Bearer " + token})
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
+ return nil, &ConnectorValidationError{Message: "Feishu permission denied while downloading a Wiki file"}
+ }
+ if resp.StatusCode >= 400 {
+ code, message := feishuErrorEnvelope(resp)
+ if code != 0 {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu API error %d (HTTP %d) while downloading a Wiki file: %s", code, resp.StatusCode, message)}
+ }
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu HTTP %d while downloading a Wiki file", resp.StatusCode)}
+ }
+
+ if declared := resp.Header.Get("Content-Length"); declared != "" {
+ if declaredSize, parseErr := strconv.ParseInt(strings.TrimSpace(declared), 10, 64); parseErr == nil && declaredSize > c.maxFileSizeBytes {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu Wiki file size exceeds the configured %d-byte limit", c.maxFileSizeBytes)}
+ }
+ }
+
+ // Feishu can return HTTP 200 with a JSON error envelope (non-zero code)
+ // instead of file bytes. A legitimate JSON file comes back with code 0 and
+ // is returned as content below.
+ if contentType := resp.Header.Get("Content-Type"); strings.HasPrefix(strings.ToLower(strings.TrimSpace(contentType)), "application/json") {
+ body, readErr := io.ReadAll(io.LimitReader(resp.Body, c.maxFileSizeBytes+1))
+ if readErr != nil {
+ return nil, &ConnectorValidationError{Message: "Failed to read a Feishu Wiki file"}
+ }
+ if int64(len(body)) > c.maxFileSizeBytes {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu Wiki file size exceeds the configured %d-byte limit", c.maxFileSizeBytes)}
+ }
+ var envelope struct {
+ Code int `json:"code"`
+ Msg string `json:"msg"`
+ }
+ if json.Unmarshal(body, &envelope) == nil && envelope.Code != 0 {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu returned a JSON error envelope instead of Wiki file content (code %d): %s", envelope.Code, envelope.Msg)}
+ }
+ return body, nil
+ }
+
+ blob, readErr := io.ReadAll(io.LimitReader(resp.Body, c.maxFileSizeBytes+1))
+ if readErr != nil {
+ return nil, &ConnectorValidationError{Message: "Failed to download a Feishu Wiki file"}
+ }
+ if int64(len(blob)) > c.maxFileSizeBytes {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu Wiki file size exceeds the configured %d-byte limit", c.maxFileSizeBytes)}
+ }
+ return blob, nil
+}
+
+// feishuDoRequest performs one request with bounded 429/5xx retry, honoring
+// Feishu's x-ogw-ratelimit-reset (and Retry-After) headers.
+func (c *FeishuWikiConnector) feishuDoRequest(ctx context.Context, method, endpoint string, body []byte, headers map[string]string) (*http.Response, error) {
+ delay := feishuRetryBaseDelay
+ var lastErr error
+ rateWaits := 0
+ for attempt := 0; attempt < feishuRetryTries; {
+ resp, err := c.feishuSend(ctx, method, endpoint, body, headers)
+ if err != nil {
+ lastErr = err
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ if attempt == feishuRetryTries-1 {
+ break
+ }
+ if !feishuSleep(ctx, delay+time.Duration(rand.Float64()*float64(feishuRetryJitter))) {
+ return nil, ctx.Err()
+ }
+ delay = time.Duration(float64(delay) * feishuRetryBackoff)
+ if delay > feishuRetryMaxDelay {
+ delay = feishuRetryMaxDelay
+ }
+ attempt++
+ continue
+ }
+ if resp.StatusCode == http.StatusTooManyRequests {
+ resp.Body.Close()
+ if rateWaits >= feishu429MaxWaits {
+ lastErr = &RateLimitTriedTooManyTimesError{Message: "Feishu rate limit exceeded while requesting " + endpoint}
+ break
+ }
+ rateWaits++
+ retryAfter := feishu429DefaultWait
+ if raw := resp.Header.Get("x-ogw-ratelimit-reset"); raw != "" {
+ if seconds, parseErr := strconv.Atoi(strings.TrimSpace(raw)); parseErr == nil && seconds >= 0 {
+ retryAfter = time.Duration(seconds) * time.Second
+ }
+ } else if raw := resp.Header.Get("Retry-After"); raw != "" {
+ if seconds, parseErr := strconv.Atoi(strings.TrimSpace(raw)); parseErr == nil && seconds >= 0 {
+ retryAfter = time.Duration(seconds) * time.Second
+ }
+ }
+ if !feishuSleep(ctx, retryAfter) {
+ return nil, ctx.Err()
+ }
+ continue
+ }
+ if resp.StatusCode >= 500 {
+ resp.Body.Close()
+ lastErr = fmt.Errorf("Feishu HTTP %d while requesting %s", resp.StatusCode, endpoint)
+ if attempt == feishuRetryTries-1 {
+ break
+ }
+ if !feishuSleep(ctx, delay+time.Duration(rand.Float64()*float64(feishuRetryJitter))) {
+ return nil, ctx.Err()
+ }
+ delay = time.Duration(float64(delay) * feishuRetryBackoff)
+ if delay > feishuRetryMaxDelay {
+ delay = feishuRetryMaxDelay
+ }
+ attempt++
+ continue
+ }
+ return resp, nil
+ }
+ if lastErr == nil {
+ lastErr = fmt.Errorf("Feishu request failed: %s", endpoint)
+ }
+ return nil, lastErr
+}
+
+// feishuSend performs one HTTP request without retry.
+func (c *FeishuWikiConnector) feishuSend(ctx context.Context, method, endpoint string, body []byte, headers map[string]string) (*http.Response, error) {
+ req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewReader(body))
+ if err != nil {
+ return nil, err
+ }
+ for key, value := range headers {
+ req.Header.Set(key, value)
+ }
+ if len(body) > 0 {
+ req.Header.Set("Content-Type", "application/json")
+ }
+ return c.httpClient.Do(req)
+}
+
+func feishuSleep(ctx context.Context, duration time.Duration) bool {
+ select {
+ case <-ctx.Done():
+ return false
+ case <-time.After(duration):
+ return true
+ }
+}
+
+// feishuDecodeResponse decodes a JSON API response and enforces Feishu error
+// semantics (permission/HTTP/business codes surfaced with code and msg).
+func feishuDecodeResponse(resp *http.Response, operation string) (map[string]any, error) {
+ defer resp.Body.Close()
+ body, err := io.ReadAll(io.LimitReader(resp.Body, feishuMaxResponseBody+1))
+ if err != nil {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Failed to read Feishu response while attempting to %s", operation)}
+ }
+ var payload map[string]any
+ if err := json.Unmarshal(body, &payload); err != nil || payload == nil {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu returned an invalid response while attempting to %s", operation)}
+ }
+ code := feishuCode(payload["code"])
+ message := feishuString(payload["msg"])
+ if message == "" {
+ message = "unknown Feishu API error"
+ }
+ if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {
+ detail := ""
+ if message != "unknown Feishu API error" {
+ detail = ": " + message
+ }
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu permission denied (HTTP %d) while attempting to %s%s", resp.StatusCode, operation, detail)}
+ }
+ if resp.StatusCode >= 400 {
+ if code != 0 {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu API error %d (HTTP %d) while attempting to %s: %s", code, resp.StatusCode, operation, message)}
+ }
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu HTTP %d while attempting to %s", resp.StatusCode, operation)}
+ }
+ if code != 0 {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu API error %d while attempting to %s: %s", code, operation, message)}
+ }
+ return payload, nil
+}
+
+// feishuErrorEnvelope reads a response body (bounded) and returns the Feishu
+// business code and message for diagnostics.
+func feishuErrorEnvelope(resp *http.Response) (int, string) {
+ body, err := io.ReadAll(io.LimitReader(resp.Body, feishuMaxResponseBody+1))
+ if err != nil {
+ return 0, ""
+ }
+ var envelope struct {
+ Code int `json:"code"`
+ Msg string `json:"msg"`
+ }
+ if json.Unmarshal(body, &envelope) != nil {
+ return 0, ""
+ }
+ return envelope.Code, envelope.Msg
+}
+
+func feishuString(value any) string {
+ if value == nil {
+ return ""
+ }
+ return strings.TrimSpace(fmt.Sprint(value))
+}
+
+func feishuCode(value any) int {
+ switch typed := value.(type) {
+ case float64:
+ return int(typed)
+ case int:
+ return typed
+ case int64:
+ return int(typed)
+ case string:
+ typed = strings.TrimSpace(typed)
+ if typed == "" {
+ return 0
+ }
+ if parsed, err := strconv.Atoi(typed); err == nil {
+ return parsed
+ }
+ }
+ return 0
+}
+
+func feishuIntDefault(value any, fallback int) int {
+ switch typed := value.(type) {
+ case float64:
+ return int(typed)
+ case int:
+ return typed
+ case int64:
+ return int(typed)
+ case string:
+ if parsed, err := strconv.Atoi(strings.TrimSpace(typed)); err == nil {
+ return parsed
+ }
+ }
+ return fallback
+}
+
+// feishuConfigInt parses a positive integer config value; empty/nil uses the
+// fallback and an invalid non-empty value returns an error.
+func feishuConfigInt(value any, fallback int) (int, error) {
+ if value == nil {
+ return fallback, nil
+ }
+ switch typed := value.(type) {
+ case string:
+ typed = strings.TrimSpace(typed)
+ if typed == "" {
+ return fallback, nil
+ }
+ parsed, err := strconv.Atoi(typed)
+ if err != nil {
+ return 0, err
+ }
+ return parsed, nil
+ case int:
+ return typed, nil
+ case int64:
+ return int(typed), nil
+ case float64:
+ return int(typed), nil
+ }
+ return 0, fmt.Errorf("invalid integer %v", value)
+}
+
+// feishuConfigInt64 parses an int64 config value; empty/nil uses the fallback.
+func feishuConfigInt64(value any, fallback int64) (int64, error) {
+ if value == nil {
+ return fallback, nil
+ }
+ switch typed := value.(type) {
+ case string:
+ typed = strings.TrimSpace(typed)
+ if typed == "" {
+ return fallback, nil
+ }
+ parsed, err := strconv.ParseInt(typed, 10, 64)
+ if err != nil {
+ return 0, err
+ }
+ return parsed, nil
+ case int:
+ return int64(typed), nil
+ case int64:
+ return typed, nil
+ case float64:
+ return int64(typed), nil
+ }
+ return 0, fmt.Errorf("invalid integer %v", value)
+}
+
+// feishuNormalizeStrings flattens a comma string or a []any value into trimmed,
+// casefolded strings.
+func feishuNormalizeStrings(value any) []string {
+ var raw []string
+ switch typed := value.(type) {
+ case string:
+ raw = strings.Split(typed, ",")
+ case []any:
+ for _, item := range typed {
+ raw = append(raw, fmt.Sprint(item))
+ }
+ case []string:
+ raw = typed
+ }
+ out := make([]string, 0, len(raw))
+ for _, item := range raw {
+ item = strings.TrimSpace(item)
+ if item == "" {
+ continue
+ }
+ out = append(out, strings.ToLower(item))
+ }
+ return out
+}
+
+func feishuExtensionSet(value any) map[string]struct{} {
+ set := map[string]struct{}{}
+ for _, term := range feishuNormalizeStrings(value) {
+ term = strings.TrimLeft(term, ".")
+ if term == "" {
+ continue
+ }
+ set["."+term] = struct{}{}
+ }
+ return set
+}
+
+func feishuKeywordSet(value any) map[string]struct{} {
+ set := map[string]struct{}{}
+ for _, term := range feishuNormalizeStrings(value) {
+ set[term] = struct{}{}
+ }
+ return set
+}
+
+func feishuParseTimestamp(value any) (time.Time, bool) {
+ if value == nil {
+ return time.Time{}, false
+ }
+ text := strings.TrimSpace(fmt.Sprint(value))
+ if text == "" {
+ return time.Time{}, false
+ }
+ if seconds, err := strconv.ParseFloat(text, 64); err == nil {
+ return time.Unix(int64(seconds), 0).UTC(), true
+ }
+ if parsed, err := time.Parse(time.RFC3339, text); err == nil {
+ return parsed.UTC(), true
+ }
+ return time.Time{}, false
+}
diff --git a/internal/syncer/connector/feishu_wiki_test.go b/internal/syncer/connector/feishu_wiki_test.go
new file mode 100644
index 0000000000..a2c2537391
--- /dev/null
+++ b/internal/syncer/connector/feishu_wiki_test.go
@@ -0,0 +1,654 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+)
+
+type feishuWikiTestServer struct {
+ server *httptest.Server
+ mu sync.Mutex
+ authCalls int
+ downloads []string
+}
+
+func (s *feishuWikiTestServer) authCount() int {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return s.authCalls
+}
+
+func (s *feishuWikiTestServer) downloadCount() int {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return len(s.downloads)
+}
+
+func newFeishuWikiTestServer(t *testing.T, handle func(w http.ResponseWriter, r *http.Request)) *feishuWikiTestServer {
+ t.Helper()
+ fixture := &feishuWikiTestServer{}
+ fixture.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal" {
+ fixture.mu.Lock()
+ fixture.authCalls++
+ fixture.mu.Unlock()
+ feishuWriteJSON(w, map[string]any{"code": 0, "msg": "success", "tenant_access_token": "test-token", "expire": 7200})
+ return
+ }
+ handle(w, r)
+ }))
+ t.Cleanup(fixture.server.Close)
+ return fixture
+}
+
+func feishuWikiBaseConfig() map[string]any {
+ return map[string]any{
+ "credentials": map[string]any{"app_id": "cli_test", "app_secret": "secret"},
+ "space_id": "space-1",
+ "root_node_token": "wikcnroot",
+ }
+}
+
+func newFeishuWikiTestConnector(t *testing.T, server *httptest.Server, config map[string]any) *FeishuWikiConnector {
+ t.Helper()
+ merged := feishuWikiBaseConfig()
+ for key, value := range config {
+ merged[key] = value
+ }
+ connector, err := NewFeishuWikiConnector(merged)
+ if err != nil {
+ t.Fatalf("NewFeishuWikiConnector: %v", err)
+ }
+ connector.baseURL = server.URL
+ connector.httpClient = server.Client()
+ return connector
+}
+
+func feishuWriteJSON(w http.ResponseWriter, payload any) {
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(payload)
+}
+
+func feishuListNode(nodeToken, objType, objToken, title string, hasChild bool, editTime any) map[string]any {
+ node := map[string]any{
+ "node_token": nodeToken,
+ "obj_type": objType,
+ "obj_token": objToken,
+ "title": title,
+ "has_child": hasChild,
+ }
+ if editTime != nil {
+ node["obj_edit_time"] = editTime
+ }
+ return node
+}
+
+func feishuListResponse(items []map[string]any, hasMore bool, pageToken string) map[string]any {
+ return map[string]any{
+ "code": 0,
+ "msg": "success",
+ "data": map[string]any{"items": items, "has_more": hasMore, "page_token": pageToken},
+ }
+}
+
+func feishuListEndpoint(r *http.Request) (string, string, string) {
+ query := r.URL.Query()
+ return query.Get("parent_node_token"), query.Get("page_token"), query.Get("space_id")
+}
+
+func TestFeishuWikiConnectorConfigValidation(t *testing.T) {
+ base := func() map[string]any { return feishuWikiBaseConfig() }
+
+ tests := []struct {
+ name string
+ config func() map[string]any
+ wantError string
+ missing *ConnectorMissingCredentialError
+ invalid *ConnectorValidationError
+ }{
+ {"missing credentials", func() map[string]any {
+ cfg := base()
+ cfg["credentials"] = map[string]any{}
+ return cfg
+ }, "Feishu Wiki", &ConnectorMissingCredentialError{}, nil},
+ {"missing space id", func() map[string]any {
+ cfg := base()
+ cfg["space_id"] = ""
+ return cfg
+ }, "space ID", nil, &ConnectorValidationError{}},
+ {"missing root node", func() map[string]any {
+ cfg := base()
+ cfg["root_node_token"] = ""
+ return cfg
+ }, "root node token", nil, &ConnectorValidationError{}},
+ {"batch size zero", func() map[string]any {
+ cfg := base()
+ cfg["batch_size"] = 0
+ return cfg
+ }, "batch_size", nil, &ConnectorValidationError{}},
+ {"batch size too large", func() map[string]any {
+ cfg := base()
+ cfg["batch_size"] = 11
+ return cfg
+ }, "batch_size", nil, &ConnectorValidationError{}},
+ {"batch size not integer", func() map[string]any {
+ cfg := base()
+ cfg["batch_size"] = "abc"
+ return cfg
+ }, "batch_size", nil, &ConnectorValidationError{}},
+ {"max size zero", func() map[string]any {
+ cfg := base()
+ cfg["max_file_size_bytes"] = 0
+ return cfg
+ }, "max_file_size_bytes", nil, &ConnectorValidationError{}},
+ {"unsupported extension", func() map[string]any {
+ cfg := base()
+ cfg["include_extensions"] = []string{"exe"}
+ return cfg
+ }, "Unsupported Feishu Wiki file extension", nil, &ConnectorValidationError{}},
+ }
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ _, err := NewFeishuWikiConnector(test.config())
+ if err == nil {
+ t.Fatalf("expected an error, got nil")
+ }
+ if !strings.Contains(err.Error(), test.wantError) {
+ t.Fatalf("error = %q, want substring %q", err.Error(), test.wantError)
+ }
+ if test.missing != nil && !errors.As(err, &test.missing) {
+ t.Fatalf("error type = %T, want *ConnectorMissingCredentialError", err)
+ }
+ if test.invalid != nil && !errors.As(err, &test.invalid) {
+ t.Fatalf("error type = %T, want *ConnectorValidationError", err)
+ }
+ })
+ }
+}
+
+func TestFeishuWikiConnectorDefaultsAndNormalization(t *testing.T) {
+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ feishuWriteJSON(w, feishuListResponse(nil, false, ""))
+ })
+ connector := newFeishuWikiTestConnector(t, server.server, map[string]any{
+ "include_extensions": []any{"pdf", ".docx", "PDF"},
+ "include_keywords": "project, handbook",
+ "exclude_keywords": []any{"draft"},
+ })
+ if connector.batchSize != defaultFeishuBatchSize {
+ t.Fatalf("batch_size = %d, want %d", connector.batchSize, defaultFeishuBatchSize)
+ }
+ if connector.maxFileSizeBytes != defaultFeishuMaxFileSize {
+ t.Fatalf("max_file_size_bytes = %d, want %d", connector.maxFileSizeBytes, defaultFeishuMaxFileSize)
+ }
+ for _, ext := range []string{".pdf", ".docx"} {
+ if _, ok := connector.includeExtensions[ext]; !ok {
+ t.Fatalf("include_extensions missing %q: %v", ext, connector.includeExtensions)
+ }
+ }
+ if _, ok := connector.includeKeywords["handbook"]; !ok {
+ t.Fatalf("include_keywords missing handbook: %v", connector.includeKeywords)
+ }
+ if _, ok := connector.excludeKeywords["draft"]; !ok {
+ t.Fatalf("exclude_keywords missing draft: %v", connector.excludeKeywords)
+ }
+}
+
+func TestFeishuWikiAuthTokenCaching(t *testing.T) {
+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ feishuWriteJSON(w, feishuListResponse(nil, false, ""))
+ })
+ connector := newFeishuWikiTestConnector(t, server.server, nil)
+
+ token, err := connector.getAccessToken(context.Background())
+ if err != nil {
+ t.Fatalf("getAccessToken: %v", err)
+ }
+ if token != "test-token" {
+ t.Fatalf("token = %q, want test-token", token)
+ }
+ if server.authCount() != 1 {
+ t.Fatalf("auth calls = %d, want 1", server.authCount())
+ }
+ // Cached token must not re-authenticate.
+ _, _ = connector.getAccessToken(context.Background())
+ if server.authCount() != 1 {
+ t.Fatalf("auth calls after cache = %d, want 1", server.authCount())
+ }
+ // Expired token must refresh.
+ connector.accessTokenExpiresAt = time.Now().Add(-time.Minute)
+ _, _ = connector.getAccessToken(context.Background())
+ if server.authCount() != 2 {
+ t.Fatalf("auth calls after expiry = %d, want 2", server.authCount())
+ }
+}
+
+func TestFeishuWikiCollectFilesPaginationAndRecursion(t *testing.T) {
+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ if !strings.HasPrefix(r.URL.Path, "/open-apis/wiki/v2/spaces/") {
+ t.Errorf("unexpected path %s", r.URL.Path)
+ http.NotFound(w, r)
+ return
+ }
+ parent, pageToken, _ := feishuListEndpoint(r)
+ switch parent {
+ case "wikcnroot":
+ if pageToken == "" {
+ feishuWriteJSON(w, feishuListResponse([]map[string]any{
+ feishuListNode("n1", "file", "o1", "a.pdf", false, "1700000000"),
+ feishuListNode("n2", "docx", "odoc", "native doc", true, "1700000001"),
+ }, true, "p2"))
+ return
+ }
+ feishuWriteJSON(w, feishuListResponse([]map[string]any{
+ feishuListNode("n3", "file", "o3", "b.docx", false, "1700000002"),
+ }, false, ""))
+ case "n2":
+ feishuWriteJSON(w, feishuListResponse([]map[string]any{
+ feishuListNode("n4", "file", "o4", "nested.pdf", false, "1700000003"),
+ }, false, ""))
+ default:
+ feishuWriteJSON(w, feishuListResponse(nil, false, ""))
+ }
+ })
+ connector := newFeishuWikiTestConnector(t, server.server, nil)
+
+ files, err := connector.collectFiles(context.Background())
+ if err != nil {
+ t.Fatalf("collectFiles: %v", err)
+ }
+ if len(files) != 3 {
+ t.Fatalf("collected %d files, want 3: %+v", len(files), files)
+ }
+ // n2 is a non-file node and is not collected, but its child n4 is.
+ got := []string{files[0].nodeToken, files[1].nodeToken, files[2].nodeToken}
+ want := []string{"n1", "n3", "n4"}
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("collected order = %v, want %v", got, want)
+ }
+ }
+ if !files[2].hasUpdatedAt || !files[2].updatedAt.Equal(time.Unix(1700000003, 0).UTC()) {
+ t.Fatalf("nested file updatedAt = %v (has=%v), want 1700000003", files[2].updatedAt, files[2].hasUpdatedAt)
+ }
+}
+
+func TestFeishuWikiFilters(t *testing.T) {
+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ feishuWriteJSON(w, feishuListResponse(nil, false, ""))
+ })
+ connector := newFeishuWikiTestConnector(t, server.server, map[string]any{
+ "include_extensions": []any{"pdf"},
+ "include_keywords": []any{"handbook"},
+ "exclude_keywords": []any{"draft"},
+ })
+
+ if !connector.matchesFilters("Project Handbook.pdf") {
+ t.Fatalf("expected handbook.pdf to match")
+ }
+ if connector.matchesFilters("Other.pdf") {
+ t.Fatalf("expected Other.pdf to be excluded by include_keywords")
+ }
+ if connector.matchesFilters("Handbook Draft.pdf") {
+ t.Fatalf("expected Handbook Draft.pdf to be excluded by exclude_keywords")
+ }
+ if connector.matchesFilters("Handbook.docx") {
+ t.Fatalf("expected Handbook.docx to be excluded by include_extensions")
+ }
+ if connector.matchesFilters("Handbook.exe") {
+ t.Fatalf("expected Handbook.exe to be excluded by supported extensions")
+ }
+}
+
+func TestFeishuWikiWindowFiltering(t *testing.T) {
+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ feishuWriteJSON(w, feishuListResponse([]map[string]any{
+ feishuListNode("before", "file", "o-before", "before.pdf", false, "1700000000"),
+ feishuListNode("inside", "file", "o-inside", "inside.pdf", false, "1700000100"),
+ feishuListNode("after", "file", "o-after", "after.pdf", false, "1700000200"),
+ feishuListNode("missing", "file", "o-missing", "missing.pdf", false, nil),
+ }, false, ""))
+ })
+ connector := newFeishuWikiTestConnector(t, server.server, nil)
+
+ start := time.Unix(1700000050, 0).UTC()
+ end := time.Unix(1700000150, 0).UTC()
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowStart: &start,
+ WindowEnd: end,
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ feishuSession := session.(*feishuWikiSyncSession)
+ if len(feishuSession.files) != 2 {
+ t.Fatalf("accepted %d files, want 2 (inside + missing timestamp): %+v", len(feishuSession.files), feishuSession.files)
+ }
+}
+
+func TestFeishuWikiWindowEndOnly(t *testing.T) {
+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ feishuWriteJSON(w, feishuListResponse([]map[string]any{
+ feishuListNode("before", "file", "o-before", "before.pdf", false, "1700000000"),
+ feishuListNode("after", "file", "o-after", "after.pdf", false, "1700000200"),
+ feishuListNode("missing", "file", "o-missing", "missing.pdf", false, nil),
+ }, false, ""))
+ })
+ connector := newFeishuWikiTestConnector(t, server.server, nil)
+
+ end := time.Unix(1700000150, 0).UTC()
+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ feishuSession := session.(*feishuWikiSyncSession)
+ if len(feishuSession.files) != 2 {
+ t.Fatalf("accepted %d files, want 2 (before end + missing timestamp): %+v", len(feishuSession.files), feishuSession.files)
+ }
+}
+
+func TestFeishuWikiDownload(t *testing.T) {
+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ if !strings.HasPrefix(r.URL.Path, "/open-apis/drive/v1/files/") {
+ http.NotFound(w, r)
+ return
+ }
+ if got := r.Header.Get("Authorization"); got != "Bearer test-token" {
+ t.Errorf("Authorization = %q, want Bearer test-token", got)
+ }
+ objectToken := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/open-apis/drive/v1/files/"), "/download")
+ switch objectToken {
+ case "o-good":
+ w.Header().Set("Content-Type", "application/octet-stream")
+ _, _ = io.WriteString(w, "file-bytes")
+ case "o-declared-oversize":
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Header().Set("Content-Length", "999999999")
+ _, _ = io.WriteString(w, "file-bytes")
+ case "o-body-oversize":
+ w.Header().Set("Content-Type", "application/octet-stream")
+ _, _ = io.WriteString(w, strings.Repeat("x", 51*1024*1024))
+ case "o-forbidden":
+ w.WriteHeader(http.StatusForbidden)
+ feishuWriteJSON(w, map[string]any{"code": 99991663, "msg": "permission denied"})
+ case "o-http-error":
+ w.WriteHeader(http.StatusBadRequest)
+ feishuWriteJSON(w, map[string]any{"code": 131006, "msg": "invalid param"})
+ case "o-json-envelope":
+ w.Header().Set("Content-Type", "application/json")
+ feishuWriteJSON(w, map[string]any{"code": 1061045, "msg": "rate limit exceeded"})
+ case "o-json-file":
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = io.WriteString(w, `{"hello":"world"}`)
+ default:
+ http.NotFound(w, r)
+ }
+ })
+ connector := newFeishuWikiTestConnector(t, server.server, nil)
+ ctx := context.Background()
+
+ if blob, err := connector.downloadFile(ctx, "o-good"); err != nil || string(blob) != "file-bytes" {
+ t.Fatalf("download good = %q, %v", blob, err)
+ }
+ if _, err := connector.downloadFile(ctx, "o-declared-oversize"); err == nil || !strings.Contains(err.Error(), "size exceeds") {
+ t.Fatalf("declared oversize error = %v", err)
+ }
+ if _, err := connector.downloadFile(ctx, "o-body-oversize"); err == nil || !strings.Contains(err.Error(), "size exceeds") {
+ t.Fatalf("body oversize error = %v", err)
+ }
+ if _, err := connector.downloadFile(ctx, "o-forbidden"); err == nil || !strings.Contains(err.Error(), "permission denied") {
+ t.Fatalf("forbidden error = %v", err)
+ }
+ if _, err := connector.downloadFile(ctx, "o-http-error"); err == nil || !strings.Contains(err.Error(), "API error 131006") || !strings.Contains(err.Error(), "invalid param") {
+ t.Fatalf("http error = %v", err)
+ }
+ if _, err := connector.downloadFile(ctx, "o-json-envelope"); err == nil || !strings.Contains(err.Error(), "JSON error envelope") {
+ t.Fatalf("json envelope error = %v", err)
+ }
+ if blob, err := connector.downloadFile(ctx, "o-json-file"); err != nil || string(blob) != `{"hello":"world"}` {
+ t.Fatalf("json file download = %q, %v", blob, err)
+ }
+}
+
+func TestFeishuWikiSyncSessionBatchingAndResume(t *testing.T) {
+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ feishuWriteJSON(w, feishuListResponse([]map[string]any{
+ feishuListNode("n1", "file", "o1", "a.pdf", false, "1700000000"),
+ feishuListNode("n2", "file", "o2", "b.pdf", false, "1700000001"),
+ feishuListNode("n3", "file", "o3", "c.pdf", false, "1700000002"),
+ feishuListNode("n4", "file", "o4", "d.pdf", false, "1700000003"),
+ feishuListNode("n5", "file", "o5", "e.pdf", false, "1700000004"),
+ }, false, ""))
+ })
+ connector := newFeishuWikiTestConnector(t, server.server, map[string]any{"batch_size": 2})
+
+ end := time.Now().UTC()
+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ feishuSession := session.(*feishuWikiSyncSession)
+ if len(feishuSession.files) != 5 {
+ t.Fatalf("accepted %d files, want 5", len(feishuSession.files))
+ }
+
+ batch, err := feishuSession.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch 1: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("batch 1 size = %d, want 2", len(batch.Documents))
+ }
+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != "feishu_wiki:space-1:n2" {
+ t.Fatalf("batch 1 checkpoint = %+v", batch.Checkpoint)
+ }
+ for i, document := range batch.Documents {
+ if len(document.Blob) == 0 {
+ t.Fatalf("batch 1 doc %d has no blob", i)
+ }
+ if document.SemanticIdentifier == "" || document.Fingerprint == "" {
+ t.Fatalf("batch 1 doc %d missing identifier/fingerprint", i)
+ }
+ }
+
+ batch2, err := feishuSession.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch 2: %v", err)
+ }
+ if len(batch2.Documents) != 2 || batch2.Checkpoint.SourceID != "feishu_wiki:space-1:n4" {
+ t.Fatalf("batch 2 = %d docs, checkpoint %+v", len(batch2.Documents), batch2.Checkpoint)
+ }
+
+ batch3, err := feishuSession.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch 3: %v", err)
+ }
+ if len(batch3.Documents) != 1 || batch3.Checkpoint.SourceID != "feishu_wiki:space-1:n5" {
+ t.Fatalf("batch 3 = %d docs, checkpoint %+v", len(batch3.Documents), batch3.Checkpoint)
+ }
+ if _, err := feishuSession.NextBatch(context.Background()); err != io.EOF {
+ t.Fatalf("NextBatch after end = %v, want io.EOF", err)
+ }
+
+ // Resume from the batch-2 checkpoint advances past the committed docs.
+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowEnd: end,
+ Resume: batch2.Checkpoint,
+ })
+ if err != nil {
+ t.Fatalf("OpenSync with resume: %v", err)
+ }
+ resumedSession := resumed.(*feishuWikiSyncSession)
+ if resumedSession.index != 4 {
+ t.Fatalf("resume index = %d, want 4", resumedSession.index)
+ }
+
+ // An anchor that is no longer present forces a window restart.
+ _, err = connector.OpenSync(context.Background(), SyncRequest{
+ WindowEnd: end,
+ Resume: &SyncCheckpoint{SourceID: "feishu_wiki:space-1:gone"},
+ })
+ if err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume with missing anchor = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
+func TestFeishuWikiOpenPruneUnsupported(t *testing.T) {
+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ feishuWriteJSON(w, feishuListResponse(nil, false, ""))
+ })
+ connector := newFeishuWikiTestConnector(t, server.server, nil)
+ _, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err == nil || !errors.Is(err, ErrPruneUnsupported) {
+ t.Fatalf("OpenPrune = %v, want ErrPruneUnsupported", err)
+ }
+}
+
+func TestFeishuWikiValidateConnects(t *testing.T) {
+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ parent, _, _ := feishuListEndpoint(r)
+ if parent != "wikcnroot" {
+ t.Errorf("validate listed parent %q, want wikcnroot", parent)
+ }
+ feishuWriteJSON(w, feishuListResponse(nil, false, ""))
+ })
+ connector := newFeishuWikiTestConnector(t, server.server, nil)
+ if err := connector.ValidateConnectorSetting(context.Background(), feishuWikiBaseConfig()); err != nil {
+ t.Fatalf("ValidateConnectorSetting: %v", err)
+ }
+ if server.authCount() != 1 {
+ t.Fatalf("auth calls = %d, want 1", server.authCount())
+ }
+}
+
+func TestFeishuWiki429Retry(t *testing.T) {
+ var server *feishuWikiTestServer
+ server = newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ if !strings.HasPrefix(r.URL.Path, "/open-apis/drive/v1/files/") {
+ http.NotFound(w, r)
+ return
+ }
+ server.mu.Lock()
+ server.downloads = append(server.downloads, r.URL.Path)
+ recorded := len(server.downloads)
+ server.mu.Unlock()
+ if recorded == 1 {
+ w.Header().Set("x-ogw-ratelimit-reset", "0")
+ w.WriteHeader(http.StatusTooManyRequests)
+ feishuWriteJSON(w, map[string]any{"code": 99991400, "msg": "request trigger frequency limit"})
+ } else {
+ w.Header().Set("Content-Type", "application/octet-stream")
+ _, _ = io.WriteString(w, "retried-bytes")
+ }
+ })
+ connector := newFeishuWikiTestConnector(t, server.server, nil)
+
+ savedWaits := feishu429MaxWaits
+ feishu429MaxWaits = 2
+ t.Cleanup(func() { feishu429MaxWaits = savedWaits })
+
+ blob, err := connector.downloadFile(context.Background(), "o-1")
+ if err != nil {
+ t.Fatalf("download after 429: %v", err)
+ }
+ if string(blob) != "retried-bytes" {
+ t.Fatalf("blob = %q", blob)
+ }
+ if server.downloadCount() != 2 {
+ t.Fatalf("download requests = %d, want 2", server.downloadCount())
+ }
+}
+
+func TestFeishuWiki429ExhaustionReturnsRateLimitError(t *testing.T) {
+ var server *feishuWikiTestServer
+ server = newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {
+ if !strings.HasPrefix(r.URL.Path, "/open-apis/drive/v1/files/") {
+ http.NotFound(w, r)
+ return
+ }
+ server.mu.Lock()
+ server.downloads = append(server.downloads, r.URL.Path)
+ server.mu.Unlock()
+ w.Header().Set("x-ogw-ratelimit-reset", "0")
+ w.WriteHeader(http.StatusTooManyRequests)
+ feishuWriteJSON(w, map[string]any{"code": 99991400, "msg": "request trigger frequency limit"})
+ })
+ connector := newFeishuWikiTestConnector(t, server.server, nil)
+
+ savedWaits := feishu429MaxWaits
+ savedTries := feishuRetryTries
+ feishu429MaxWaits = 5
+ feishuRetryTries = 3 // 429 waits must not consume these general retry attempts
+ t.Cleanup(func() {
+ feishu429MaxWaits = savedWaits
+ feishuRetryTries = savedTries
+ })
+
+ _, err := connector.downloadFile(context.Background(), "o-1")
+ var rateErr *RateLimitTriedTooManyTimesError
+ if !errors.As(err, &rateErr) {
+ t.Fatalf("err=%T want *RateLimitTriedTooManyTimesError, got %v", err, err)
+ }
+ if got := server.downloadCount(); got != feishu429MaxWaits+1 {
+ t.Fatalf("download requests = %d, want %d (429 waits must not consume general retries)", got, feishu429MaxWaits+1)
+ }
+}
+
+func TestFeishuWikiRegistryRegistration(t *testing.T) {
+ registry := NewRegistry()
+ RegisterBuiltIns(registry)
+
+ connector, err := registry.OpenFromConfig("feishu_wiki", feishuWikiBaseConfig())
+ if err != nil {
+ t.Fatalf("OpenFromConfig feishu_wiki: %v", err)
+ }
+ if _, ok := connector.(*FeishuWikiConnector); !ok {
+ t.Fatalf("connector type = %T, want *FeishuWikiConnector", connector)
+ }
+
+ _, err = registry.OpenFromConfig("feishu_wiki", map[string]any{})
+ var credErr *ConnectorMissingCredentialError
+ if err == nil || !errors.As(err, &credErr) {
+ t.Fatalf("missing-credential error = %v", err)
+ }
+ if err == nil || !strings.Contains(err.Error(), "Feishu Wiki") {
+ t.Fatalf("missing-credential message = %v", err)
+ }
+
+ // The task-context factory path must also resolve.
+ taskConnector, err := registry.Open(context.Background(), dao.SyncTaskContext{
+ Connector: entity.Connector{Source: "feishu_wiki", Config: entity.JSONMap(feishuWikiBaseConfig())},
+ })
+ if err != nil {
+ t.Fatalf("Open feishu_wiki: %v", err)
+ }
+ if _, ok := taskConnector.(*FeishuWikiConnector); !ok {
+ t.Fatalf("task connector type = %T", taskConnector)
+ }
+}
diff --git a/web/src/pages/user-setting/data-source/backend-adapter.test.ts b/web/src/pages/user-setting/data-source/backend-adapter.test.ts
index 7121f87c00..d95ab1f7bd 100644
--- a/web/src/pages/user-setting/data-source/backend-adapter.test.ts
+++ b/web/src/pages/user-setting/data-source/backend-adapter.test.ts
@@ -3,33 +3,20 @@
supports: () => false,
};
-const loadAvailableDataSourceKeys = (language: 'go' | 'python') => {
- jest.resetModules();
- jest.doMock('@/utils/backend-runtime', () => ({
- getBackendLanguage: () => language,
- subscribeBackendLanguage: () => () => undefined,
- }));
-
+const loadAvailableDataSourceKeys = () => {
const { getAvailableDataSourceKeys } = require('./backend-adapter');
return getAvailableDataSourceKeys();
};
describe('data source backend adapter', () => {
- afterEach(() => {
- jest.dontMock('@/utils/backend-runtime');
- });
-
- it('includes Feishu Wiki for the Python backend', () => {
- expect(loadAvailableDataSourceKeys('python')).toContain('feishu_wiki');
+ it('includes Feishu Wiki so the connector is available on both backends', () => {
+ expect(loadAvailableDataSourceKeys()).toContain('feishu_wiki');
});
- it('hides only Feishu Wiki for the Go backend', () => {
- const pythonKeys = loadAvailableDataSourceKeys('python');
- const goKeys = loadAvailableDataSourceKeys('go');
-
- expect(goKeys).not.toContain('feishu_wiki');
- expect(goKeys).toEqual(
- pythonKeys.filter((source: string) => source !== 'feishu_wiki'),
+ it('returns every DataSourceKey', () => {
+ const { DataSourceKey } = require('./constant');
+ expect(loadAvailableDataSourceKeys().sort()).toEqual(
+ Object.values(DataSourceKey).sort(),
);
});
});
diff --git a/web/src/pages/user-setting/data-source/backend-adapter.ts b/web/src/pages/user-setting/data-source/backend-adapter.ts
index e018c3167a..70c57ca2fc 100644
--- a/web/src/pages/user-setting/data-source/backend-adapter.ts
+++ b/web/src/pages/user-setting/data-source/backend-adapter.ts
@@ -1,13 +1,4 @@
-import { pickByBackend } from '@/utils/backend-variant';
import { DataSourceKey } from './constant';
-const PythonDataSourceKeys = Object.values(DataSourceKey);
-const GoDataSourceKeys = PythonDataSourceKeys.filter(
- (source) => source !== DataSourceKey.FEISHU_WIKI,
-);
-
export const getAvailableDataSourceKeys = (): DataSourceKey[] =>
- pickByBackend({
- go: GoDataSourceKeys,
- python: PythonDataSourceKeys,
- });
+ Object.values(DataSourceKey);