RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(syncer): add Feishu Wiki connector for the Go backend (#19453)
mkaaad3e1396d0422c
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 68144a4beb..0dbeaba817 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -27,6 +27,7 @@ import (// and the raw-config factory (used by the test-connection endpoint).func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "confluence", NewConfluenceConnector)+ registerBuiltIn(registry, "feishu_wiki", NewFeishuWikiConnector)registerBuiltIn(registry, "rss", NewRSSConnector)registerBuiltIn(registry, "sitemap", NewSitemapConnector)registerBuiltIn(registry, "salesforce", NewSalesforceConnector)diff --git a/internal/syncer/connector/feishu_wiki.go b/internal/syncer/connector/feishu_wiki.gonew file mode 100644index 0000000000..c4583a1ee5--- /dev/null+++ b/internal/syncer/connector/feishu_wiki.go@@ -0,0 +1,880 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "bytes"+ "context"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "fmt"+ "io"+ "math/rand"+ "net/http"+ "net/url"+ "path"+ "sort"+ "strconv"+ "strings"+ "sync"+ "time"+)++const (+ feishuOpenBaseURL = "https://open.feishu.cn"+ feishuRequestTimeout = 60 * time.Second+ defaultFeishuBatchSize = 2+ maxFeishuBatchSize = 10+ feishuNodePageSize = 50+ defaultFeishuMaxFileSize = 50 * 1024 * 1024+ feishuMaxResponseBody = 4 * 1024 * 1024+)++// feishuSupportedExtensions is the set of downloadable Wiki file extensions the+// connector accepts, mirroring the Python connector's allowlist.+var feishuSupportedExtensions = map[string]struct{}{+ ".csv": {}, ".doc": {}, ".docx": {}, ".eml": {}, ".gif": {}, ".html": {},+ ".jpeg": {}, ".jpg": {}, ".json": {}, ".md": {}, ".mdx": {}, ".pdf": {},+ ".png": {}, ".ppt": {}, ".pptx": {}, ".tif": {}, ".txt": {}, ".xls": {},+ ".xlsx": {},+}++// Retry/backoff knobs for Feishu API calls. They are package variables so tests+// can shrink the delays (mirrors rest_api.go).+var (+ feishuRetryTries = 5+ feishuRetryBaseDelay = time.Second+ feishuRetryMaxDelay = 30 * time.Second+ feishuRetryBackoff = 2.0+ feishuRetryJitter = time.Second+ feishu429MaxWaits = 5+ feishu429DefaultWait = 5 * time.Second+)++// ErrFeishuWikiPruneUnsupported reports that the Feishu Wiki connector cannot+// reconcile deletions; prune tasks then complete without deleting anything.+var ErrFeishuWikiPruneUnsupported = fmt.Errorf("feishu wiki connector does not support prune: %w", ErrPruneUnsupported)++// FeishuWikiConnector reads downloadable attachment files from one Feishu Wiki+// subtree using an application's tenant credentials.+type FeishuWikiConnector struct {+ appID string+ appSecret string+ spaceID string+ rootNodeToken string+ includeExtensions map[string]struct{}+ includeKeywords map[string]struct{}+ excludeKeywords map[string]struct{}+ batchSize int+ maxFileSizeBytes int64+ baseURL string+ httpClient *http.Client++ mu sync.Mutex+ accessToken string+ accessTokenExpiresAt time.Time+}++// NewFeishuWikiConnector creates a Feishu Wiki connector from Python-compatible+// config. It performs synchronous config validation but no network I/O.+func NewFeishuWikiConnector(config map[string]any) (*FeishuWikiConnector, error) {+ credentials, _ := config["credentials"].(map[string]any)+ batchSize, err := feishuConfigInt(config["batch_size"], defaultFeishuBatchSize)+ if err != nil {+ return nil, &ConnectorValidationError{Message: "Feishu Wiki batch_size must be an integer"}+ }+ maxFileSizeBytes, err := feishuConfigInt64(config["max_file_size_bytes"], defaultFeishuMaxFileSize)+ if err != nil {+ return nil, &ConnectorValidationError{Message: "Feishu Wiki max_file_size_bytes must be an integer"}+ }+ connector := &FeishuWikiConnector{+ appID: strings.TrimSpace(stringConfig(credentials["app_id"])),+ appSecret: strings.TrimSpace(stringConfig(credentials["app_secret"])),+ spaceID: strings.TrimSpace(stringConfig(config["space_id"])),+ rootNodeToken: strings.TrimSpace(stringConfig(config["root_node_token"])),+ includeExtensions: feishuExtensionSet(config["include_extensions"]),+ includeKeywords: feishuKeywordSet(config["include_keywords"]),+ excludeKeywords: feishuKeywordSet(config["exclude_keywords"]),+ batchSize: batchSize,+ maxFileSizeBytes: maxFileSizeBytes,+ baseURL: feishuOpenBaseURL,+ httpClient: &http.Client{Timeout: feishuRequestTimeout},+ }+ if err := connector.validateConfig(); err != nil {+ return nil, err+ }+ return connector, nil+}++// validateConfig validates configuration values without network I/O.+func (c *FeishuWikiConnector) validateConfig() error {+ if c.appID == "" || c.appSecret == "" {+ return &ConnectorMissingCredentialError{Message: "Feishu Wiki"}+ }+ if c.spaceID == "" {+ return &ConnectorValidationError{Message: "Feishu Wiki space ID is required"}+ }+ if c.rootNodeToken == "" {+ return &ConnectorValidationError{Message: "Feishu Wiki root node token is required"}+ }+ if c.batchSize < 1 || c.batchSize > maxFeishuBatchSize {+ return &ConnectorValidationError{Message: fmt.Sprintf("Feishu Wiki batch_size must be between 1 and %d", maxFeishuBatchSize)}+ }+ if c.maxFileSizeBytes <= 0 {+ return &ConnectorValidationError{Message: "Feishu Wiki max_file_size_bytes must be a positive integer"}+ }+ for ext := range c.includeExtensions {+ if _, ok := feishuSupportedExtensions[ext]; !ok {+ return &ConnectorValidationError{Message: fmt.Sprintf("Unsupported Feishu Wiki file extension: %s", ext)}+ }+ }+ return nil+}++// Validate validates configuration and credentials by authenticating and+// listing the first page of the configured root node.+func (c *FeishuWikiConnector) Validate(ctx context.Context) error {+ if err := c.validateConfig(); err != nil {+ return err+ }+ if _, err := c.getAccessToken(ctx); err != nil {+ return err+ }+ if _, _, _, err := c.listChildPage(ctx, c.rootNodeToken, ""); err != nil {+ return fmt.Errorf("Feishu Wiki validation failed: %w", err)+ }+ return nil+}++// ValidateConnectorSetting validates Feishu Wiki settings from an unsaved config.+func (c *FeishuWikiConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ candidate, err := NewFeishuWikiConnector(request)+ if err != nil {+ return err+ }+ if c != nil {+ if c.httpClient != nil {+ candidate.httpClient = c.httpClient+ }+ if c.baseURL != "" {+ candidate.baseURL = c.baseURL+ }+ }+ return candidate.Validate(ctx)+}++// OpenSync opens one Feishu Wiki synchronization session.+func (c *FeishuWikiConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ files, err := c.collectFiles(ctx)+ if err != nil {+ return nil, err+ }+ end := request.WindowEnd+ if end.IsZero() {+ end = time.Now().UTC()+ }+ accepted := make([]feishuWikiFile, 0, len(files))+ for _, file := range files {+ if request.WindowStart != nil || !request.WindowEnd.IsZero() {+ var updatedAt *time.Time+ if file.hasUpdatedAt {+ updatedAt = &file.updatedAt+ }+ if !feishuWithinWindow(updatedAt, request.WindowStart, &end) {+ continue+ }+ }+ if !c.matchesFilters(file.title) {+ continue+ }+ accepted = append(accepted, file)+ }+ sort.Slice(accepted, func(i, j int) bool {+ return accepted[i].sourceID(c.spaceID) < accepted[j].sourceID(c.spaceID)+ })+ session := &feishuWikiSyncSession{+ connector: c,+ files: accepted,+ batchSize: c.batchSize,+ windowEnd: end,+ }+ if err := session.applyResume(request.Resume); err != nil {+ return nil, err+ }+ return session, nil+}++// OpenPrune reports prune as unsupported: ordinary polling does not reconcile+// source deletions (mirroring the Python connector).+func (c *FeishuWikiConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ return nil, ErrFeishuWikiPruneUnsupported+}++// feishuWikiFile is the metadata for one downloadable Wiki file node.+type feishuWikiFile struct {+ nodeToken string+ objToken string+ title string+ objType string+ parentNodeToken string+ updatedAt time.Time+ hasUpdatedAt bool+}++func (f feishuWikiFile) sourceID(spaceID string) string {+ return "feishu_wiki:" + spaceID + ":" + f.nodeToken+}++// feishuWikiSyncSession streams accepted Wiki files in fixed-size batches,+// resuming from a committed SourceID anchor.+type feishuWikiSyncSession struct {+ connector *FeishuWikiConnector+ files []feishuWikiFile+ batchSize int+ windowEnd time.Time+ index int+}++// NextBatch returns the next source document batch.+func (s *feishuWikiSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ if s.index >= len(s.files) {+ return SyncBatch{}, io.EOF+ }+ documents := make([]SourceDocument, 0, s.batchSize)+ var last feishuWikiFile+ for attempts := 0; attempts < s.batchSize && s.index < len(s.files); attempts++ {+ file := s.files[s.index]+ s.index+++ last = file+ document, err := s.connector.buildDocument(ctx, file, s.windowEnd)+ if err != nil {+ return SyncBatch{}, err+ }+ documents = append(documents, document)+ }+ return SyncBatch{Documents: documents, Checkpoint: feishuWikiCheckpoint(s.connector.spaceID, last)}, nil+}++// Close releases connector resources.+func (s *feishuWikiSyncSession) Close() error {+ return nil+}++// applyResume advances the session past the last committed SourceID anchor.+func (s *feishuWikiSyncSession) applyResume(checkpoint *SyncCheckpoint) error {+ if checkpoint == nil {+ return nil+ }+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)+ if sourceID == "" {+ return fmt.Errorf("feishu wiki sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)+ }+ for index, file := range s.files {+ if file.sourceID(s.connector.spaceID) == sourceID {+ s.index = index + 1+ return nil+ }+ }+ return fmt.Errorf("feishu wiki resume anchor %q was not found in the current listing: %w", sourceID, ErrSyncResumeInvalid)+}++func feishuWikiCheckpoint(spaceID string, file feishuWikiFile) *SyncCheckpoint {+ checkpoint := &SyncCheckpoint{+ Cursor: file.sourceID(spaceID),+ SourceID: file.sourceID(spaceID),+ }+ if file.hasUpdatedAt {+ updatedAt := file.updatedAt+ checkpoint.UpdatedAt = &updatedAt+ }+ return checkpoint+}++// buildDocument downloads one file and maps it to a SourceDocument.+func (c *FeishuWikiConnector) buildDocument(ctx context.Context, file feishuWikiFile, fallbackUpdatedAt time.Time) (SourceDocument, error) {+ blob, err := c.downloadFile(ctx, file.objToken)+ if err != nil {+ return SourceDocument{}, err+ }+ updatedAt := fallbackUpdatedAt+ if file.hasUpdatedAt {+ updatedAt = file.updatedAt+ }+ parentNodeToken := file.parentNodeToken+ if parentNodeToken == "" {+ parentNodeToken = c.rootNodeToken+ }+ sha := sha256.Sum256(blob)+ shaHex := hex.EncodeToString(sha[:])+ return SourceDocument{+ SourceID: file.sourceID(c.spaceID),+ SemanticIdentifier: firstNonEmpty(file.title, file.nodeToken),+ Extension: strings.ToLower(path.Ext(file.title)),+ Blob: blob,+ UpdatedAt: updatedAt,+ SizeBytes: int64(len(blob)),+ Fingerprint: shaHex[:32],+ Metadata: map[string]any{+ "source": "feishu_wiki",+ "wiki_space_id": c.spaceID,+ "wiki_node_token": file.nodeToken,+ "wiki_object_token": file.objToken,+ "wiki_object_type": file.objType,+ "wiki_parent_node_token": parentNodeToken,+ "wiki_url": fmt.Sprintf("https://feishu.cn/wiki/%s", file.nodeToken),+ "source_updated_at": updatedAt.UTC().Format(time.RFC3339),+ "content_sha256": shaHex,+ },+ }, nil+}++// matchesFilters applies the extension and keyword filters to a node title.+func (c *FeishuWikiConnector) matchesFilters(title string) bool {+ ext := strings.ToLower(path.Ext(title))+ if _, ok := feishuSupportedExtensions[ext]; !ok {+ return false+ }+ if len(c.includeExtensions) > 0 {+ if _, ok := c.includeExtensions[ext]; !ok {+ return false+ }+ }+ normalizedTitle := strings.ToLower(title)+ if len(c.includeKeywords) > 0 && !feishuContainsAny(normalizedTitle, c.includeKeywords) {+ return false+ }+ return !feishuContainsAny(normalizedTitle, c.excludeKeywords)+}++func feishuContainsAny(value string, terms map[string]struct{}) bool {+ for term := range terms {+ if strings.Contains(value, term) {+ return true+ }+ }+ return false+}++// feishuWithinWindow reports whether a node update time falls in (start, end].+// Nodes without a timestamp are always included so content is not silently+// omitted.+func feishuWithinWindow(updatedAt *time.Time, start, end *time.Time) bool {+ if updatedAt == nil {+ return true+ }+ if start != nil && !start.Before(*updatedAt) {+ return false+ }+ return end == nil || !updatedAt.After(*end)+}++// collectFiles walks the configured Wiki subtree and returns all downloadable+// file nodes (filters are applied later so the window can be applied at open).+func (c *FeishuWikiConnector) collectFiles(ctx context.Context) ([]feishuWikiFile, error) {+ var files []feishuWikiFile+ stack := []string{c.rootNodeToken}+ for len(stack) > 0 {+ parent := stack[len(stack)-1]+ stack = stack[:len(stack)-1]+ pageToken := ""+ for {+ items, hasMore, next, err := c.listChildPage(ctx, parent, pageToken)+ if err != nil {+ return nil, err+ }+ for _, node := range items {+ nodeToken := feishuString(node["node_token"])+ objType := feishuString(node["obj_type"])+ objToken := feishuString(node["obj_token"])+ title := feishuString(node["title"])+ hasChild, _ := node["has_child"].(bool)+ if objType == "file" {+ updatedAt, hasUpdatedAt := feishuParseTimestamp(node["obj_edit_time"])+ files = append(files, feishuWikiFile{+ nodeToken: nodeToken,+ objToken: objToken,+ title: title,+ objType: objType,+ parentNodeToken: feishuString(node["parent_node_token"]),+ updatedAt: updatedAt,+ hasUpdatedAt: hasUpdatedAt,+ })+ }+ if hasChild && nodeToken != "" {+ stack = append(stack, nodeToken)+ }+ }+ if !hasMore {+ break+ }+ if next == "" {+ return nil, &ConnectorValidationError{Message: "Feishu Wiki pagination indicated more data without a page token"}+ }+ pageToken = next+ }+ }+ return files, nil+}++// listChildPage lists one page of child nodes for a parent node.+func (c *FeishuWikiConnector) listChildPage(ctx context.Context, parentNodeToken, pageToken string) ([]map[string]any, bool, string, error) {+ token, err := c.getAccessToken(ctx)+ if err != nil {+ return nil, false, "", err+ }+ endpoint := fmt.Sprintf("%s/open-apis/wiki/v2/spaces/%s/nodes?page_size=%d", c.baseURL, url.PathEscape(c.spaceID), feishuNodePageSize)+ if parentNodeToken != "" {+ endpoint += "&parent_node_token=" + url.QueryEscape(parentNodeToken)+ }+ if pageToken != "" {+ endpoint += "&page_token=" + url.QueryEscape(pageToken)+ }+ resp, err := c.feishuDoRequest(ctx, http.MethodGet, endpoint, nil, map[string]string{"Authorization": "Bearer " + token})+ if err != nil {+ return nil, false, "", err+ }+ payload, err := feishuDecodeResponse(resp, "list Wiki nodes")+ if err != nil {+ return nil, false, "", err+ }+ data, _ := payload["data"].(map[string]any)+ if data == nil {+ return nil, false, "", &ConnectorValidationError{Message: "Feishu Wiki node response contained invalid data"}+ }+ items := feishuNodeItems(data["items"])+ hasMore, _ := data["has_more"].(bool)+ return items, hasMore, feishuString(data["page_token"]), nil+}++func feishuNodeItems(value any) []map[string]any {+ raw, ok := value.([]any)+ if !ok {+ return nil+ }+ items := make([]map[string]any, 0, len(raw))+ for _, item := range raw {+ if node, ok := item.(map[string]any); ok {+ items = append(items, node)+ }+ }+ return items+}++// getAccessToken returns a cached tenant access token, refreshing it when+// expired or absent.+func (c *FeishuWikiConnector) getAccessToken(ctx context.Context) (string, error) {+ c.mu.Lock()+ defer c.mu.Unlock()+ if c.accessToken != "" && time.Now().Before(c.accessTokenExpiresAt) {+ return c.accessToken, nil+ }+ body, err := json.Marshal(map[string]string{"app_id": c.appID, "app_secret": c.appSecret})+ if err != nil {+ return "", err+ }+ resp, err := c.feishuDoRequest(ctx, http.MethodPost, c.baseURL+"/open-apis/auth/v3/tenant_access_token/internal", body, nil)+ if err != nil {+ return "", err+ }+ payload, err := feishuDecodeResponse(resp, "authenticate")+ if err != nil {+ return "", err+ }+ accessToken := feishuString(payload["tenant_access_token"])+ if accessToken == "" {+ return "", &ConnectorValidationError{Message: "Feishu authentication response did not contain a tenant access token"}+ }+ expiresIn := feishuIntDefault(payload["expire"], 7200)+ if expiresIn < 60 {+ expiresIn = 7200+ }+ c.accessToken = accessToken+ c.accessTokenExpiresAt = time.Now().Add(time.Duration(expiresIn-60) * time.Second)+ return accessToken, nil+}++// downloadFile streams one Wiki file, enforcing the configured size limit and+// rejecting Feishu JSON error envelopes so they are never ingested as content.+func (c *FeishuWikiConnector) downloadFile(ctx context.Context, objectToken string) ([]byte, error) {+ token, err := c.getAccessToken(ctx)+ if err != nil {+ return nil, err+ }+ endpoint := fmt.Sprintf("%s/open-apis/drive/v1/files/%s/download", c.baseURL, url.PathEscape(objectToken))+ resp, err := c.feishuDoRequest(ctx, http.MethodGet, endpoint, nil, map[string]string{"Authorization": "Bearer " + token})+ if err != nil {+ return nil, err+ }+ defer resp.Body.Close()++ if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {+ return nil, &ConnectorValidationError{Message: "Feishu permission denied while downloading a Wiki file"}+ }+ if resp.StatusCode >= 400 {+ code, message := feishuErrorEnvelope(resp)+ if code != 0 {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu API error %d (HTTP %d) while downloading a Wiki file: %s", code, resp.StatusCode, message)}+ }+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu HTTP %d while downloading a Wiki file", resp.StatusCode)}+ }++ if declared := resp.Header.Get("Content-Length"); declared != "" {+ if declaredSize, parseErr := strconv.ParseInt(strings.TrimSpace(declared), 10, 64); parseErr == nil && declaredSize > c.maxFileSizeBytes {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu Wiki file size exceeds the configured %d-byte limit", c.maxFileSizeBytes)}+ }+ }++ // Feishu can return HTTP 200 with a JSON error envelope (non-zero code)+ // instead of file bytes. A legitimate JSON file comes back with code 0 and+ // is returned as content below.+ if contentType := resp.Header.Get("Content-Type"); strings.HasPrefix(strings.ToLower(strings.TrimSpace(contentType)), "application/json") {+ body, readErr := io.ReadAll(io.LimitReader(resp.Body, c.maxFileSizeBytes+1))+ if readErr != nil {+ return nil, &ConnectorValidationError{Message: "Failed to read a Feishu Wiki file"}+ }+ if int64(len(body)) > c.maxFileSizeBytes {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu Wiki file size exceeds the configured %d-byte limit", c.maxFileSizeBytes)}+ }+ var envelope struct {+ Code int `json:"code"`+ Msg string `json:"msg"`+ }+ if json.Unmarshal(body, &envelope) == nil && envelope.Code != 0 {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu returned a JSON error envelope instead of Wiki file content (code %d): %s", envelope.Code, envelope.Msg)}+ }+ return body, nil+ }++ blob, readErr := io.ReadAll(io.LimitReader(resp.Body, c.maxFileSizeBytes+1))+ if readErr != nil {+ return nil, &ConnectorValidationError{Message: "Failed to download a Feishu Wiki file"}+ }+ if int64(len(blob)) > c.maxFileSizeBytes {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu Wiki file size exceeds the configured %d-byte limit", c.maxFileSizeBytes)}+ }+ return blob, nil+}++// feishuDoRequest performs one request with bounded 429/5xx retry, honoring+// Feishu's x-ogw-ratelimit-reset (and Retry-After) headers.+func (c *FeishuWikiConnector) feishuDoRequest(ctx context.Context, method, endpoint string, body []byte, headers map[string]string) (*http.Response, error) {+ delay := feishuRetryBaseDelay+ var lastErr error+ rateWaits := 0+ for attempt := 0; attempt < feishuRetryTries; {+ resp, err := c.feishuSend(ctx, method, endpoint, body, headers)+ if err != nil {+ lastErr = err+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }+ if attempt == feishuRetryTries-1 {+ break+ }+ if !feishuSleep(ctx, delay+time.Duration(rand.Float64()*float64(feishuRetryJitter))) {+ return nil, ctx.Err()+ }+ delay = time.Duration(float64(delay) * feishuRetryBackoff)+ if delay > feishuRetryMaxDelay {+ delay = feishuRetryMaxDelay+ }+ attempt+++ continue+ }+ if resp.StatusCode == http.StatusTooManyRequests {+ resp.Body.Close()+ if rateWaits >= feishu429MaxWaits {+ lastErr = &RateLimitTriedTooManyTimesError{Message: "Feishu rate limit exceeded while requesting " + endpoint}+ break+ }+ rateWaits+++ retryAfter := feishu429DefaultWait+ if raw := resp.Header.Get("x-ogw-ratelimit-reset"); raw != "" {+ if seconds, parseErr := strconv.Atoi(strings.TrimSpace(raw)); parseErr == nil && seconds >= 0 {+ retryAfter = time.Duration(seconds) * time.Second+ }+ } else if raw := resp.Header.Get("Retry-After"); raw != "" {+ if seconds, parseErr := strconv.Atoi(strings.TrimSpace(raw)); parseErr == nil && seconds >= 0 {+ retryAfter = time.Duration(seconds) * time.Second+ }+ }+ if !feishuSleep(ctx, retryAfter) {+ return nil, ctx.Err()+ }+ continue+ }+ if resp.StatusCode >= 500 {+ resp.Body.Close()+ lastErr = fmt.Errorf("Feishu HTTP %d while requesting %s", resp.StatusCode, endpoint)+ if attempt == feishuRetryTries-1 {+ break+ }+ if !feishuSleep(ctx, delay+time.Duration(rand.Float64()*float64(feishuRetryJitter))) {+ return nil, ctx.Err()+ }+ delay = time.Duration(float64(delay) * feishuRetryBackoff)+ if delay > feishuRetryMaxDelay {+ delay = feishuRetryMaxDelay+ }+ attempt+++ continue+ }+ return resp, nil+ }+ if lastErr == nil {+ lastErr = fmt.Errorf("Feishu request failed: %s", endpoint)+ }+ return nil, lastErr+}++// feishuSend performs one HTTP request without retry.+func (c *FeishuWikiConnector) feishuSend(ctx context.Context, method, endpoint string, body []byte, headers map[string]string) (*http.Response, error) {+ req, err := http.NewRequestWithContext(ctx, method, endpoint, bytes.NewReader(body))+ if err != nil {+ return nil, err+ }+ for key, value := range headers {+ req.Header.Set(key, value)+ }+ if len(body) > 0 {+ req.Header.Set("Content-Type", "application/json")+ }+ return c.httpClient.Do(req)+}++func feishuSleep(ctx context.Context, duration time.Duration) bool {+ select {+ case <-ctx.Done():+ return false+ case <-time.After(duration):+ return true+ }+}++// feishuDecodeResponse decodes a JSON API response and enforces Feishu error+// semantics (permission/HTTP/business codes surfaced with code and msg).+func feishuDecodeResponse(resp *http.Response, operation string) (map[string]any, error) {+ defer resp.Body.Close()+ body, err := io.ReadAll(io.LimitReader(resp.Body, feishuMaxResponseBody+1))+ if err != nil {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Failed to read Feishu response while attempting to %s", operation)}+ }+ var payload map[string]any+ if err := json.Unmarshal(body, &payload); err != nil || payload == nil {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu returned an invalid response while attempting to %s", operation)}+ }+ code := feishuCode(payload["code"])+ message := feishuString(payload["msg"])+ if message == "" {+ message = "unknown Feishu API error"+ }+ if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden {+ detail := ""+ if message != "unknown Feishu API error" {+ detail = ": " + message+ }+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu permission denied (HTTP %d) while attempting to %s%s", resp.StatusCode, operation, detail)}+ }+ if resp.StatusCode >= 400 {+ if code != 0 {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu API error %d (HTTP %d) while attempting to %s: %s", code, resp.StatusCode, operation, message)}+ }+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu HTTP %d while attempting to %s", resp.StatusCode, operation)}+ }+ if code != 0 {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Feishu API error %d while attempting to %s: %s", code, operation, message)}+ }+ return payload, nil+}++// feishuErrorEnvelope reads a response body (bounded) and returns the Feishu+// business code and message for diagnostics.+func feishuErrorEnvelope(resp *http.Response) (int, string) {+ body, err := io.ReadAll(io.LimitReader(resp.Body, feishuMaxResponseBody+1))+ if err != nil {+ return 0, ""+ }+ var envelope struct {+ Code int `json:"code"`+ Msg string `json:"msg"`+ }+ if json.Unmarshal(body, &envelope) != nil {+ return 0, ""+ }+ return envelope.Code, envelope.Msg+}++func feishuString(value any) string {+ if value == nil {+ return ""+ }+ return strings.TrimSpace(fmt.Sprint(value))+}++func feishuCode(value any) int {+ switch typed := value.(type) {+ case float64:+ return int(typed)+ case int:+ return typed+ case int64:+ return int(typed)+ case string:+ typed = strings.TrimSpace(typed)+ if typed == "" {+ return 0+ }+ if parsed, err := strconv.Atoi(typed); err == nil {+ return parsed+ }+ }+ return 0+}++func feishuIntDefault(value any, fallback int) int {+ switch typed := value.(type) {+ case float64:+ return int(typed)+ case int:+ return typed+ case int64:+ return int(typed)+ case string:+ if parsed, err := strconv.Atoi(strings.TrimSpace(typed)); err == nil {+ return parsed+ }+ }+ return fallback+}++// feishuConfigInt parses a positive integer config value; empty/nil uses the+// fallback and an invalid non-empty value returns an error.+func feishuConfigInt(value any, fallback int) (int, error) {+ if value == nil {+ return fallback, nil+ }+ switch typed := value.(type) {+ case string:+ typed = strings.TrimSpace(typed)+ if typed == "" {+ return fallback, nil+ }+ parsed, err := strconv.Atoi(typed)+ if err != nil {+ return 0, err+ }+ return parsed, nil+ case int:+ return typed, nil+ case int64:+ return int(typed), nil+ case float64:+ return int(typed), nil+ }+ return 0, fmt.Errorf("invalid integer %v", value)+}++// feishuConfigInt64 parses an int64 config value; empty/nil uses the fallback.+func feishuConfigInt64(value any, fallback int64) (int64, error) {+ if value == nil {+ return fallback, nil+ }+ switch typed := value.(type) {+ case string:+ typed = strings.TrimSpace(typed)+ if typed == "" {+ return fallback, nil+ }+ parsed, err := strconv.ParseInt(typed, 10, 64)+ if err != nil {+ return 0, err+ }+ return parsed, nil+ case int:+ return int64(typed), nil+ case int64:+ return typed, nil+ case float64:+ return int64(typed), nil+ }+ return 0, fmt.Errorf("invalid integer %v", value)+}++// feishuNormalizeStrings flattens a comma string or a []any value into trimmed,+// casefolded strings.+func feishuNormalizeStrings(value any) []string {+ var raw []string+ switch typed := value.(type) {+ case string:+ raw = strings.Split(typed, ",")+ case []any:+ for _, item := range typed {+ raw = append(raw, fmt.Sprint(item))+ }+ case []string:+ raw = typed+ }+ out := make([]string, 0, len(raw))+ for _, item := range raw {+ item = strings.TrimSpace(item)+ if item == "" {+ continue+ }+ out = append(out, strings.ToLower(item))+ }+ return out+}++func feishuExtensionSet(value any) map[string]struct{} {+ set := map[string]struct{}{}+ for _, term := range feishuNormalizeStrings(value) {+ term = strings.TrimLeft(term, ".")+ if term == "" {+ continue+ }+ set["."+term] = struct{}{}+ }+ return set+}++func feishuKeywordSet(value any) map[string]struct{} {+ set := map[string]struct{}{}+ for _, term := range feishuNormalizeStrings(value) {+ set[term] = struct{}{}+ }+ return set+}++func feishuParseTimestamp(value any) (time.Time, bool) {+ if value == nil {+ return time.Time{}, false+ }+ text := strings.TrimSpace(fmt.Sprint(value))+ if text == "" {+ return time.Time{}, false+ }+ if seconds, err := strconv.ParseFloat(text, 64); err == nil {+ return time.Unix(int64(seconds), 0).UTC(), true+ }+ if parsed, err := time.Parse(time.RFC3339, text); err == nil {+ return parsed.UTC(), true+ }+ return time.Time{}, false+}diff --git a/internal/syncer/connector/feishu_wiki_test.go b/internal/syncer/connector/feishu_wiki_test.gonew file mode 100644index 0000000000..a2c2537391--- /dev/null+++ b/internal/syncer/connector/feishu_wiki_test.go@@ -0,0 +1,654 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+ "strings"+ "sync"+ "testing"+ "time"+)++type feishuWikiTestServer struct {+ server *httptest.Server+ mu sync.Mutex+ authCalls int+ downloads []string+}++func (s *feishuWikiTestServer) authCount() int {+ s.mu.Lock()+ defer s.mu.Unlock()+ return s.authCalls+}++func (s *feishuWikiTestServer) downloadCount() int {+ s.mu.Lock()+ defer s.mu.Unlock()+ return len(s.downloads)+}++func newFeishuWikiTestServer(t *testing.T, handle func(w http.ResponseWriter, r *http.Request)) *feishuWikiTestServer {+ t.Helper()+ fixture := &feishuWikiTestServer{}+ fixture.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path == "/open-apis/auth/v3/tenant_access_token/internal" {+ fixture.mu.Lock()+ fixture.authCalls+++ fixture.mu.Unlock()+ feishuWriteJSON(w, map[string]any{"code": 0, "msg": "success", "tenant_access_token": "test-token", "expire": 7200})+ return+ }+ handle(w, r)+ }))+ t.Cleanup(fixture.server.Close)+ return fixture+}++func feishuWikiBaseConfig() map[string]any {+ return map[string]any{+ "credentials": map[string]any{"app_id": "cli_test", "app_secret": "secret"},+ "space_id": "space-1",+ "root_node_token": "wikcnroot",+ }+}++func newFeishuWikiTestConnector(t *testing.T, server *httptest.Server, config map[string]any) *FeishuWikiConnector {+ t.Helper()+ merged := feishuWikiBaseConfig()+ for key, value := range config {+ merged[key] = value+ }+ connector, err := NewFeishuWikiConnector(merged)+ if err != nil {+ t.Fatalf("NewFeishuWikiConnector: %v", err)+ }+ connector.baseURL = server.URL+ connector.httpClient = server.Client()+ return connector+}++func feishuWriteJSON(w http.ResponseWriter, payload any) {+ w.Header().Set("Content-Type", "application/json")+ _ = json.NewEncoder(w).Encode(payload)+}++func feishuListNode(nodeToken, objType, objToken, title string, hasChild bool, editTime any) map[string]any {+ node := map[string]any{+ "node_token": nodeToken,+ "obj_type": objType,+ "obj_token": objToken,+ "title": title,+ "has_child": hasChild,+ }+ if editTime != nil {+ node["obj_edit_time"] = editTime+ }+ return node+}++func feishuListResponse(items []map[string]any, hasMore bool, pageToken string) map[string]any {+ return map[string]any{+ "code": 0,+ "msg": "success",+ "data": map[string]any{"items": items, "has_more": hasMore, "page_token": pageToken},+ }+}++func feishuListEndpoint(r *http.Request) (string, string, string) {+ query := r.URL.Query()+ return query.Get("parent_node_token"), query.Get("page_token"), query.Get("space_id")+}++func TestFeishuWikiConnectorConfigValidation(t *testing.T) {+ base := func() map[string]any { return feishuWikiBaseConfig() }++ tests := []struct {+ name string+ config func() map[string]any+ wantError string+ missing *ConnectorMissingCredentialError+ invalid *ConnectorValidationError+ }{+ {"missing credentials", func() map[string]any {+ cfg := base()+ cfg["credentials"] = map[string]any{}+ return cfg+ }, "Feishu Wiki", &ConnectorMissingCredentialError{}, nil},+ {"missing space id", func() map[string]any {+ cfg := base()+ cfg["space_id"] = ""+ return cfg+ }, "space ID", nil, &ConnectorValidationError{}},+ {"missing root node", func() map[string]any {+ cfg := base()+ cfg["root_node_token"] = ""+ return cfg+ }, "root node token", nil, &ConnectorValidationError{}},+ {"batch size zero", func() map[string]any {+ cfg := base()+ cfg["batch_size"] = 0+ return cfg+ }, "batch_size", nil, &ConnectorValidationError{}},+ {"batch size too large", func() map[string]any {+ cfg := base()+ cfg["batch_size"] = 11+ return cfg+ }, "batch_size", nil, &ConnectorValidationError{}},+ {"batch size not integer", func() map[string]any {+ cfg := base()+ cfg["batch_size"] = "abc"+ return cfg+ }, "batch_size", nil, &ConnectorValidationError{}},+ {"max size zero", func() map[string]any {+ cfg := base()+ cfg["max_file_size_bytes"] = 0+ return cfg+ }, "max_file_size_bytes", nil, &ConnectorValidationError{}},+ {"unsupported extension", func() map[string]any {+ cfg := base()+ cfg["include_extensions"] = []string{"exe"}+ return cfg+ }, "Unsupported Feishu Wiki file extension", nil, &ConnectorValidationError{}},+ }+ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ _, err := NewFeishuWikiConnector(test.config())+ if err == nil {+ t.Fatalf("expected an error, got nil")+ }+ if !strings.Contains(err.Error(), test.wantError) {+ t.Fatalf("error = %q, want substring %q", err.Error(), test.wantError)+ }+ if test.missing != nil && !errors.As(err, &test.missing) {+ t.Fatalf("error type = %T, want *ConnectorMissingCredentialError", err)+ }+ if test.invalid != nil && !errors.As(err, &test.invalid) {+ t.Fatalf("error type = %T, want *ConnectorValidationError", err)+ }+ })+ }+}++func TestFeishuWikiConnectorDefaultsAndNormalization(t *testing.T) {+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ feishuWriteJSON(w, feishuListResponse(nil, false, ""))+ })+ connector := newFeishuWikiTestConnector(t, server.server, map[string]any{+ "include_extensions": []any{"pdf", ".docx", "PDF"},+ "include_keywords": "project, handbook",+ "exclude_keywords": []any{"draft"},+ })+ if connector.batchSize != defaultFeishuBatchSize {+ t.Fatalf("batch_size = %d, want %d", connector.batchSize, defaultFeishuBatchSize)+ }+ if connector.maxFileSizeBytes != defaultFeishuMaxFileSize {+ t.Fatalf("max_file_size_bytes = %d, want %d", connector.maxFileSizeBytes, defaultFeishuMaxFileSize)+ }+ for _, ext := range []string{".pdf", ".docx"} {+ if _, ok := connector.includeExtensions[ext]; !ok {+ t.Fatalf("include_extensions missing %q: %v", ext, connector.includeExtensions)+ }+ }+ if _, ok := connector.includeKeywords["handbook"]; !ok {+ t.Fatalf("include_keywords missing handbook: %v", connector.includeKeywords)+ }+ if _, ok := connector.excludeKeywords["draft"]; !ok {+ t.Fatalf("exclude_keywords missing draft: %v", connector.excludeKeywords)+ }+}++func TestFeishuWikiAuthTokenCaching(t *testing.T) {+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ feishuWriteJSON(w, feishuListResponse(nil, false, ""))+ })+ connector := newFeishuWikiTestConnector(t, server.server, nil)++ token, err := connector.getAccessToken(context.Background())+ if err != nil {+ t.Fatalf("getAccessToken: %v", err)+ }+ if token != "test-token" {+ t.Fatalf("token = %q, want test-token", token)+ }+ if server.authCount() != 1 {+ t.Fatalf("auth calls = %d, want 1", server.authCount())+ }+ // Cached token must not re-authenticate.+ _, _ = connector.getAccessToken(context.Background())+ if server.authCount() != 1 {+ t.Fatalf("auth calls after cache = %d, want 1", server.authCount())+ }+ // Expired token must refresh.+ connector.accessTokenExpiresAt = time.Now().Add(-time.Minute)+ _, _ = connector.getAccessToken(context.Background())+ if server.authCount() != 2 {+ t.Fatalf("auth calls after expiry = %d, want 2", server.authCount())+ }+}++func TestFeishuWikiCollectFilesPaginationAndRecursion(t *testing.T) {+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ if !strings.HasPrefix(r.URL.Path, "/open-apis/wiki/v2/spaces/") {+ t.Errorf("unexpected path %s", r.URL.Path)+ http.NotFound(w, r)+ return+ }+ parent, pageToken, _ := feishuListEndpoint(r)+ switch parent {+ case "wikcnroot":+ if pageToken == "" {+ feishuWriteJSON(w, feishuListResponse([]map[string]any{+ feishuListNode("n1", "file", "o1", "a.pdf", false, "1700000000"),+ feishuListNode("n2", "docx", "odoc", "native doc", true, "1700000001"),+ }, true, "p2"))+ return+ }+ feishuWriteJSON(w, feishuListResponse([]map[string]any{+ feishuListNode("n3", "file", "o3", "b.docx", false, "1700000002"),+ }, false, ""))+ case "n2":+ feishuWriteJSON(w, feishuListResponse([]map[string]any{+ feishuListNode("n4", "file", "o4", "nested.pdf", false, "1700000003"),+ }, false, ""))+ default:+ feishuWriteJSON(w, feishuListResponse(nil, false, ""))+ }+ })+ connector := newFeishuWikiTestConnector(t, server.server, nil)++ files, err := connector.collectFiles(context.Background())+ if err != nil {+ t.Fatalf("collectFiles: %v", err)+ }+ if len(files) != 3 {+ t.Fatalf("collected %d files, want 3: %+v", len(files), files)+ }+ // n2 is a non-file node and is not collected, but its child n4 is.+ got := []string{files[0].nodeToken, files[1].nodeToken, files[2].nodeToken}+ want := []string{"n1", "n3", "n4"}+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("collected order = %v, want %v", got, want)+ }+ }+ if !files[2].hasUpdatedAt || !files[2].updatedAt.Equal(time.Unix(1700000003, 0).UTC()) {+ t.Fatalf("nested file updatedAt = %v (has=%v), want 1700000003", files[2].updatedAt, files[2].hasUpdatedAt)+ }+}++func TestFeishuWikiFilters(t *testing.T) {+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ feishuWriteJSON(w, feishuListResponse(nil, false, ""))+ })+ connector := newFeishuWikiTestConnector(t, server.server, map[string]any{+ "include_extensions": []any{"pdf"},+ "include_keywords": []any{"handbook"},+ "exclude_keywords": []any{"draft"},+ })++ if !connector.matchesFilters("Project Handbook.pdf") {+ t.Fatalf("expected handbook.pdf to match")+ }+ if connector.matchesFilters("Other.pdf") {+ t.Fatalf("expected Other.pdf to be excluded by include_keywords")+ }+ if connector.matchesFilters("Handbook Draft.pdf") {+ t.Fatalf("expected Handbook Draft.pdf to be excluded by exclude_keywords")+ }+ if connector.matchesFilters("Handbook.docx") {+ t.Fatalf("expected Handbook.docx to be excluded by include_extensions")+ }+ if connector.matchesFilters("Handbook.exe") {+ t.Fatalf("expected Handbook.exe to be excluded by supported extensions")+ }+}++func TestFeishuWikiWindowFiltering(t *testing.T) {+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ feishuWriteJSON(w, feishuListResponse([]map[string]any{+ feishuListNode("before", "file", "o-before", "before.pdf", false, "1700000000"),+ feishuListNode("inside", "file", "o-inside", "inside.pdf", false, "1700000100"),+ feishuListNode("after", "file", "o-after", "after.pdf", false, "1700000200"),+ feishuListNode("missing", "file", "o-missing", "missing.pdf", false, nil),+ }, false, ""))+ })+ connector := newFeishuWikiTestConnector(t, server.server, nil)++ start := time.Unix(1700000050, 0).UTC()+ end := time.Unix(1700000150, 0).UTC()+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowStart: &start,+ WindowEnd: end,+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ feishuSession := session.(*feishuWikiSyncSession)+ if len(feishuSession.files) != 2 {+ t.Fatalf("accepted %d files, want 2 (inside + missing timestamp): %+v", len(feishuSession.files), feishuSession.files)+ }+}++func TestFeishuWikiWindowEndOnly(t *testing.T) {+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ feishuWriteJSON(w, feishuListResponse([]map[string]any{+ feishuListNode("before", "file", "o-before", "before.pdf", false, "1700000000"),+ feishuListNode("after", "file", "o-after", "after.pdf", false, "1700000200"),+ feishuListNode("missing", "file", "o-missing", "missing.pdf", false, nil),+ }, false, ""))+ })+ connector := newFeishuWikiTestConnector(t, server.server, nil)++ end := time.Unix(1700000150, 0).UTC()+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ feishuSession := session.(*feishuWikiSyncSession)+ if len(feishuSession.files) != 2 {+ t.Fatalf("accepted %d files, want 2 (before end + missing timestamp): %+v", len(feishuSession.files), feishuSession.files)+ }+}++func TestFeishuWikiDownload(t *testing.T) {+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ if !strings.HasPrefix(r.URL.Path, "/open-apis/drive/v1/files/") {+ http.NotFound(w, r)+ return+ }+ if got := r.Header.Get("Authorization"); got != "Bearer test-token" {+ t.Errorf("Authorization = %q, want Bearer test-token", got)+ }+ objectToken := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/open-apis/drive/v1/files/"), "/download")+ switch objectToken {+ case "o-good":+ w.Header().Set("Content-Type", "application/octet-stream")+ _, _ = io.WriteString(w, "file-bytes")+ case "o-declared-oversize":+ w.Header().Set("Content-Type", "application/octet-stream")+ w.Header().Set("Content-Length", "999999999")+ _, _ = io.WriteString(w, "file-bytes")+ case "o-body-oversize":+ w.Header().Set("Content-Type", "application/octet-stream")+ _, _ = io.WriteString(w, strings.Repeat("x", 51*1024*1024))+ case "o-forbidden":+ w.WriteHeader(http.StatusForbidden)+ feishuWriteJSON(w, map[string]any{"code": 99991663, "msg": "permission denied"})+ case "o-http-error":+ w.WriteHeader(http.StatusBadRequest)+ feishuWriteJSON(w, map[string]any{"code": 131006, "msg": "invalid param"})+ case "o-json-envelope":+ w.Header().Set("Content-Type", "application/json")+ feishuWriteJSON(w, map[string]any{"code": 1061045, "msg": "rate limit exceeded"})+ case "o-json-file":+ w.Header().Set("Content-Type", "application/json")+ _, _ = io.WriteString(w, `{"hello":"world"}`)+ default:+ http.NotFound(w, r)+ }+ })+ connector := newFeishuWikiTestConnector(t, server.server, nil)+ ctx := context.Background()++ if blob, err := connector.downloadFile(ctx, "o-good"); err != nil || string(blob) != "file-bytes" {+ t.Fatalf("download good = %q, %v", blob, err)+ }+ if _, err := connector.downloadFile(ctx, "o-declared-oversize"); err == nil || !strings.Contains(err.Error(), "size exceeds") {+ t.Fatalf("declared oversize error = %v", err)+ }+ if _, err := connector.downloadFile(ctx, "o-body-oversize"); err == nil || !strings.Contains(err.Error(), "size exceeds") {+ t.Fatalf("body oversize error = %v", err)+ }+ if _, err := connector.downloadFile(ctx, "o-forbidden"); err == nil || !strings.Contains(err.Error(), "permission denied") {+ t.Fatalf("forbidden error = %v", err)+ }+ if _, err := connector.downloadFile(ctx, "o-http-error"); err == nil || !strings.Contains(err.Error(), "API error 131006") || !strings.Contains(err.Error(), "invalid param") {+ t.Fatalf("http error = %v", err)+ }+ if _, err := connector.downloadFile(ctx, "o-json-envelope"); err == nil || !strings.Contains(err.Error(), "JSON error envelope") {+ t.Fatalf("json envelope error = %v", err)+ }+ if blob, err := connector.downloadFile(ctx, "o-json-file"); err != nil || string(blob) != `{"hello":"world"}` {+ t.Fatalf("json file download = %q, %v", blob, err)+ }+}++func TestFeishuWikiSyncSessionBatchingAndResume(t *testing.T) {+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ feishuWriteJSON(w, feishuListResponse([]map[string]any{+ feishuListNode("n1", "file", "o1", "a.pdf", false, "1700000000"),+ feishuListNode("n2", "file", "o2", "b.pdf", false, "1700000001"),+ feishuListNode("n3", "file", "o3", "c.pdf", false, "1700000002"),+ feishuListNode("n4", "file", "o4", "d.pdf", false, "1700000003"),+ feishuListNode("n5", "file", "o5", "e.pdf", false, "1700000004"),+ }, false, ""))+ })+ connector := newFeishuWikiTestConnector(t, server.server, map[string]any{"batch_size": 2})++ end := time.Now().UTC()+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ feishuSession := session.(*feishuWikiSyncSession)+ if len(feishuSession.files) != 5 {+ t.Fatalf("accepted %d files, want 5", len(feishuSession.files))+ }++ batch, err := feishuSession.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch 1: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("batch 1 size = %d, want 2", len(batch.Documents))+ }+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != "feishu_wiki:space-1:n2" {+ t.Fatalf("batch 1 checkpoint = %+v", batch.Checkpoint)+ }+ for i, document := range batch.Documents {+ if len(document.Blob) == 0 {+ t.Fatalf("batch 1 doc %d has no blob", i)+ }+ if document.SemanticIdentifier == "" || document.Fingerprint == "" {+ t.Fatalf("batch 1 doc %d missing identifier/fingerprint", i)+ }+ }++ batch2, err := feishuSession.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch 2: %v", err)+ }+ if len(batch2.Documents) != 2 || batch2.Checkpoint.SourceID != "feishu_wiki:space-1:n4" {+ t.Fatalf("batch 2 = %d docs, checkpoint %+v", len(batch2.Documents), batch2.Checkpoint)+ }++ batch3, err := feishuSession.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch 3: %v", err)+ }+ if len(batch3.Documents) != 1 || batch3.Checkpoint.SourceID != "feishu_wiki:space-1:n5" {+ t.Fatalf("batch 3 = %d docs, checkpoint %+v", len(batch3.Documents), batch3.Checkpoint)+ }+ if _, err := feishuSession.NextBatch(context.Background()); err != io.EOF {+ t.Fatalf("NextBatch after end = %v, want io.EOF", err)+ }++ // Resume from the batch-2 checkpoint advances past the committed docs.+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowEnd: end,+ Resume: batch2.Checkpoint,+ })+ if err != nil {+ t.Fatalf("OpenSync with resume: %v", err)+ }+ resumedSession := resumed.(*feishuWikiSyncSession)+ if resumedSession.index != 4 {+ t.Fatalf("resume index = %d, want 4", resumedSession.index)+ }++ // An anchor that is no longer present forces a window restart.+ _, err = connector.OpenSync(context.Background(), SyncRequest{+ WindowEnd: end,+ Resume: &SyncCheckpoint{SourceID: "feishu_wiki:space-1:gone"},+ })+ if err == nil || !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("resume with missing anchor = %v, want ErrSyncResumeInvalid", err)+ }+}++func TestFeishuWikiOpenPruneUnsupported(t *testing.T) {+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ feishuWriteJSON(w, feishuListResponse(nil, false, ""))+ })+ connector := newFeishuWikiTestConnector(t, server.server, nil)+ _, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err == nil || !errors.Is(err, ErrPruneUnsupported) {+ t.Fatalf("OpenPrune = %v, want ErrPruneUnsupported", err)+ }+}++func TestFeishuWikiValidateConnects(t *testing.T) {+ server := newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ parent, _, _ := feishuListEndpoint(r)+ if parent != "wikcnroot" {+ t.Errorf("validate listed parent %q, want wikcnroot", parent)+ }+ feishuWriteJSON(w, feishuListResponse(nil, false, ""))+ })+ connector := newFeishuWikiTestConnector(t, server.server, nil)+ if err := connector.ValidateConnectorSetting(context.Background(), feishuWikiBaseConfig()); err != nil {+ t.Fatalf("ValidateConnectorSetting: %v", err)+ }+ if server.authCount() != 1 {+ t.Fatalf("auth calls = %d, want 1", server.authCount())+ }+}++func TestFeishuWiki429Retry(t *testing.T) {+ var server *feishuWikiTestServer+ server = newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ if !strings.HasPrefix(r.URL.Path, "/open-apis/drive/v1/files/") {+ http.NotFound(w, r)+ return+ }+ server.mu.Lock()+ server.downloads = append(server.downloads, r.URL.Path)+ recorded := len(server.downloads)+ server.mu.Unlock()+ if recorded == 1 {+ w.Header().Set("x-ogw-ratelimit-reset", "0")+ w.WriteHeader(http.StatusTooManyRequests)+ feishuWriteJSON(w, map[string]any{"code": 99991400, "msg": "request trigger frequency limit"})+ } else {+ w.Header().Set("Content-Type", "application/octet-stream")+ _, _ = io.WriteString(w, "retried-bytes")+ }+ })+ connector := newFeishuWikiTestConnector(t, server.server, nil)++ savedWaits := feishu429MaxWaits+ feishu429MaxWaits = 2+ t.Cleanup(func() { feishu429MaxWaits = savedWaits })++ blob, err := connector.downloadFile(context.Background(), "o-1")+ if err != nil {+ t.Fatalf("download after 429: %v", err)+ }+ if string(blob) != "retried-bytes" {+ t.Fatalf("blob = %q", blob)+ }+ if server.downloadCount() != 2 {+ t.Fatalf("download requests = %d, want 2", server.downloadCount())+ }+}++func TestFeishuWiki429ExhaustionReturnsRateLimitError(t *testing.T) {+ var server *feishuWikiTestServer+ server = newFeishuWikiTestServer(t, func(w http.ResponseWriter, r *http.Request) {+ if !strings.HasPrefix(r.URL.Path, "/open-apis/drive/v1/files/") {+ http.NotFound(w, r)+ return+ }+ server.mu.Lock()+ server.downloads = append(server.downloads, r.URL.Path)+ server.mu.Unlock()+ w.Header().Set("x-ogw-ratelimit-reset", "0")+ w.WriteHeader(http.StatusTooManyRequests)+ feishuWriteJSON(w, map[string]any{"code": 99991400, "msg": "request trigger frequency limit"})+ })+ connector := newFeishuWikiTestConnector(t, server.server, nil)++ savedWaits := feishu429MaxWaits+ savedTries := feishuRetryTries+ feishu429MaxWaits = 5+ feishuRetryTries = 3 // 429 waits must not consume these general retry attempts+ t.Cleanup(func() {+ feishu429MaxWaits = savedWaits+ feishuRetryTries = savedTries+ })++ _, err := connector.downloadFile(context.Background(), "o-1")+ var rateErr *RateLimitTriedTooManyTimesError+ if !errors.As(err, &rateErr) {+ t.Fatalf("err=%T want *RateLimitTriedTooManyTimesError, got %v", err, err)+ }+ if got := server.downloadCount(); got != feishu429MaxWaits+1 {+ t.Fatalf("download requests = %d, want %d (429 waits must not consume general retries)", got, feishu429MaxWaits+1)+ }+}++func TestFeishuWikiRegistryRegistration(t *testing.T) {+ registry := NewRegistry()+ RegisterBuiltIns(registry)++ connector, err := registry.OpenFromConfig("feishu_wiki", feishuWikiBaseConfig())+ if err != nil {+ t.Fatalf("OpenFromConfig feishu_wiki: %v", err)+ }+ if _, ok := connector.(*FeishuWikiConnector); !ok {+ t.Fatalf("connector type = %T, want *FeishuWikiConnector", connector)+ }++ _, err = registry.OpenFromConfig("feishu_wiki", map[string]any{})+ var credErr *ConnectorMissingCredentialError+ if err == nil || !errors.As(err, &credErr) {+ t.Fatalf("missing-credential error = %v", err)+ }+ if err == nil || !strings.Contains(err.Error(), "Feishu Wiki") {+ t.Fatalf("missing-credential message = %v", err)+ }++ // The task-context factory path must also resolve.+ taskConnector, err := registry.Open(context.Background(), dao.SyncTaskContext{+ Connector: entity.Connector{Source: "feishu_wiki", Config: entity.JSONMap(feishuWikiBaseConfig())},+ })+ if err != nil {+ t.Fatalf("Open feishu_wiki: %v", err)+ }+ if _, ok := taskConnector.(*FeishuWikiConnector); !ok {+ t.Fatalf("task connector type = %T", taskConnector)+ }+}diff --git a/web/src/pages/user-setting/data-source/backend-adapter.test.ts b/web/src/pages/user-setting/data-source/backend-adapter.test.tsindex 7121f87c00..d95ab1f7bd 100644--- a/web/src/pages/user-setting/data-source/backend-adapter.test.ts+++ b/web/src/pages/user-setting/data-source/backend-adapter.test.ts@@ -3,33 +3,20 @@supports: () => false,};-const loadAvailableDataSourceKeys = (language: 'go' | 'python') => {- jest.resetModules();- jest.doMock('@/utils/backend-runtime', () => ({- getBackendLanguage: () => language,- subscribeBackendLanguage: () => () => undefined,- }));-+const loadAvailableDataSourceKeys = () => {const { getAvailableDataSourceKeys } = require('./backend-adapter');return getAvailableDataSourceKeys();};describe('data source backend adapter', () => {- afterEach(() => {- jest.dontMock('@/utils/backend-runtime');- });-- it('includes Feishu Wiki for the Python backend', () => {- expect(loadAvailableDataSourceKeys('python')).toContain('feishu_wiki');+ it('includes Feishu Wiki so the connector is available on both backends', () => {+ expect(loadAvailableDataSourceKeys()).toContain('feishu_wiki');});- it('hides only Feishu Wiki for the Go backend', () => {- const pythonKeys = loadAvailableDataSourceKeys('python');- const goKeys = loadAvailableDataSourceKeys('go');-- expect(goKeys).not.toContain('feishu_wiki');- expect(goKeys).toEqual(- pythonKeys.filter((source: string) => source !== 'feishu_wiki'),+ it('returns every DataSourceKey', () => {+ const { DataSourceKey } = require('./constant');+ expect(loadAvailableDataSourceKeys().sort()).toEqual(+ Object.values(DataSourceKey).sort(),);});});diff --git a/web/src/pages/user-setting/data-source/backend-adapter.ts b/web/src/pages/user-setting/data-source/backend-adapter.tsindex e018c3167a..70c57ca2fc 100644--- a/web/src/pages/user-setting/data-source/backend-adapter.ts+++ b/web/src/pages/user-setting/data-source/backend-adapter.ts@@ -1,13 +1,4 @@-import { pickByBackend } from '@/utils/backend-variant';import { DataSourceKey } from './constant';-const PythonDataSourceKeys = Object.values(DataSourceKey);-const GoDataSourceKeys = PythonDataSourceKeys.filter(- (source) => source !== DataSourceKey.FEISHU_WIKI,-);-export const getAvailableDataSourceKeys = (): DataSourceKey[] =>- pickByBackend({- go: GoDataSourceKeys,- python: PythonDataSourceKeys,- });+ Object.values(DataSourceKey);