Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat[syncer]: implement dingtalk AI table data source (#18539)

Haruko386a88866f6b8d3
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index 84f4066482..57efd1f03e 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -36,6 +36,7 @@ func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "google-drive", NewGoogleDriveConnector)
registerBuiltIn(registry, "google_drive", NewGoogleDriveConnector)
registerBuiltIn(registry, "google_cloud_storage", NewGoogleCloudStorageConnector)
+ registerBuiltIn(registry, "dingtalk_ai_table", NewDingTalkAITableConnector)
registerBuiltIn(registry, "imap", NewIMAPConnector)
registerBuiltIn(registry, "outlook", NewOutlookConnector)
registerBuiltIn(registry, "notion", NewNotionConnector)
diff --git a/internal/syncer/connector/dingtalk_ai_table.go b/internal/syncer/connector/dingtalk_ai_table.go
new file mode 100644
index 0000000000..9a92220b62
--- /dev/null
+++ b/internal/syncer/connector/dingtalk_ai_table.go
@@ -0,0 +1,514 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "bytes"
+ "context"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/zeebo/xxh3"
+)
+
+const (
+ defaultDingTalkAITableBatchSize = 32
+ dingTalkAITableAPIBaseURL = "https://api.dingtalk.com"
+ dingTalkAITableDocIDPrefix = "dingtalk_ai_table:"
+ dingTalkAITableRequestTimeout = 60 * time.Second
+)
+
+// DingTalkAITableConnector reads records from DingTalk AI Table.
+type DingTalkAITableConnector struct {
+ tableID string
+ operatorID string
+ accessToken string
+ batchSize int
+ apiBaseURL string
+ httpClient *http.Client
+
+ getSheets func(ctx context.Context) ([]dingTalkAITableSheet, error)
+ listRecords func(ctx context.Context, sheetID, nextToken string, maxResults int) ([]dingTalkAITableRecord, string, error)
+}
+
+// NewDingTalkAITableConnector creates a DingTalk AI Table connector from
+// Python-compatible config.
+func NewDingTalkAITableConnector(config map[string]any) (*DingTalkAITableConnector, error) {
+ credentials := configAnyMap(config["credentials"])
+ return &DingTalkAITableConnector{
+ tableID: strings.TrimSpace(stringConfig(config["table_id"])),
+ operatorID: strings.TrimSpace(stringConfig(config["operator_id"])),
+ accessToken: stringConfig(credentials["access_token"]),
+ batchSize: configInt(config["batch_size"], defaultDingTalkAITableBatchSize),
+ apiBaseURL: dingTalkAITableAPIBaseURL,
+ httpClient: &http.Client{Timeout: dingTalkAITableRequestTimeout},
+ }, nil
+}
+
+// Validate validates DingTalk AI Table settings and credentials.
+func (c *DingTalkAITableConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return fmt.Errorf("DingTalk AI Table connector is nil")
+ }
+ if c.tableID == "" {
+ return &ConnectorValidationError{Message: "DingTalk AI Table table_id is required"}
+ }
+ if c.operatorID == "" {
+ return &ConnectorValidationError{Message: "DingTalk AI Table operator_id is required"}
+ }
+ if c.accessToken == "" {
+ return &ConnectorMissingCredentialError{Message: "DingTalk access_token is required"}
+ }
+ if c.batchSize <= 0 {
+ return &ConnectorValidationError{Message: "batch_size must be a positive integer"}
+ }
+ if err := validateDingTalkAITableAPIBaseURL(c.apiBaseURL); err != nil {
+ return err
+ }
+ if _, err := c.loadSheets(ctx); err != nil {
+ return &ConnectorValidationError{Message: fmt.Sprintf("DingTalk Notable credential validation failed: %v", err)}
+ }
+ return nil
+}
+
+// ValidateConnectorSetting validates DingTalk AI Table settings from an
+// unsaved config.
+func (c *DingTalkAITableConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ candidate, err := NewDingTalkAITableConnector(request)
+ if err != nil {
+ return err
+ }
+ return candidate.Validate(ctx)
+}
+
+// OpenSync opens one DingTalk AI Table sync session.
+func (c *DingTalkAITableConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ if err := c.validateStatic(); err != nil {
+ return nil, err
+ }
+ documents, err := c.collectDocuments(ctx, request)
+ if err != nil {
+ return nil, err
+ }
+ session := &dingTalkAITableSyncSession{documents: documents, batchSize: c.batchSize}
+ session.applyResume(request.Resume)
+ return session, nil
+}
+
+// OpenPrune opens one complete DingTalk AI Table prune snapshot session.
+func (c *DingTalkAITableConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ if err := c.validateStatic(); err != nil {
+ return nil, err
+ }
+ documents, err := c.collectSlimDocuments(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &dingTalkAITablePruneSession{documents: documents, batchSize: c.batchSize}, nil
+}
+
+func (c *DingTalkAITableConnector) validateStatic() error {
+ if c == nil {
+ return fmt.Errorf("DingTalk AI Table connector is nil")
+ }
+ if c.tableID == "" {
+ return &ConnectorValidationError{Message: "DingTalk AI Table table_id is required"}
+ }
+ if c.operatorID == "" {
+ return &ConnectorValidationError{Message: "DingTalk AI Table operator_id is required"}
+ }
+ if c.accessToken == "" {
+ return &ConnectorMissingCredentialError{Message: "DingTalk access_token is required"}
+ }
+ if c.batchSize <= 0 {
+ return &ConnectorValidationError{Message: "batch_size must be a positive integer"}
+ }
+ if err := validateDingTalkAITableAPIBaseURL(c.apiBaseURL); err != nil {
+ return err
+ }
+ return nil
+}
+
+func (c *DingTalkAITableConnector) collectDocuments(ctx context.Context, request SyncRequest) ([]SourceDocument, error) {
+ sheets, err := c.loadSheets(ctx)
+ if err != nil {
+ return nil, err
+ }
+ documents := []SourceDocument{}
+ for _, sheet := range sheets {
+ records, err := c.loadRecords(ctx, sheet.ID)
+ if err != nil {
+ return nil, err
+ }
+ for _, record := range records {
+ if record.ID == "" {
+ continue
+ }
+ doc, hasUpdatedAt := c.recordDocument(sheet, record)
+ if !hasUpdatedAt && dingTalkAITableRequiresTimestamp(request) {
+ continue
+ }
+ if hasUpdatedAt && !dingTalkAITableInWindow(doc.UpdatedAt, request) {
+ continue
+ }
+ documents = append(documents, doc)
+ }
+ }
+ sort.SliceStable(documents, func(i, j int) bool {
+ return documents[i].SourceID < documents[j].SourceID
+ })
+ return documents, nil
+}
+
+func (c *DingTalkAITableConnector) collectSlimDocuments(ctx context.Context) ([]SlimDocument, error) {
+ sheets, err := c.loadSheets(ctx)
+ if err != nil {
+ return nil, err
+ }
+ documents := []SlimDocument{}
+ for _, sheet := range sheets {
+ nextToken := ""
+ for {
+ records, token, err := c.loadRecordPage(ctx, sheet.ID, nextToken, 100)
+ if err != nil {
+ return nil, err
+ }
+ for _, record := range records {
+ if record.ID != "" {
+ documents = append(documents, SlimDocument{SourceID: c.documentID(sheet.ID, record.ID)})
+ }
+ }
+ if token == "" {
+ break
+ }
+ nextToken = token
+ }
+ }
+ sort.SliceStable(documents, func(i, j int) bool {
+ return documents[i].SourceID < documents[j].SourceID
+ })
+ return documents, nil
+}
+
+func (c *DingTalkAITableConnector) loadSheets(ctx context.Context) ([]dingTalkAITableSheet, error) {
+ if c.getSheets != nil {
+ return c.getSheets(ctx)
+ }
+ values := url.Values{}
+ values.Set("operatorId", c.operatorID)
+ var response dingTalkAITableSheetsResponse
+ if err := c.doJSON(ctx, http.MethodGet, fmt.Sprintf("/v1.0/notable/bases/%s/sheets?%s", url.PathEscape(c.tableID), values.Encode()), nil, &response); err != nil {
+ return nil, err
+ }
+ sheets := make([]dingTalkAITableSheet, 0, len(response.Value))
+ for _, sheet := range response.Value {
+ if sheet.ID == "" {
+ continue
+ }
+ sheets = append(sheets, sheet)
+ }
+ return sheets, nil
+}
+
+func (c *DingTalkAITableConnector) loadRecords(ctx context.Context, sheetID string) ([]dingTalkAITableRecord, error) {
+ records := []dingTalkAITableRecord{}
+ nextToken := ""
+ for {
+ page, token, err := c.loadRecordPage(ctx, sheetID, nextToken, 100)
+ if err != nil {
+ return nil, err
+ }
+ records = append(records, page...)
+ if token == "" {
+ break
+ }
+ nextToken = token
+ }
+ return records, nil
+}
+
+func (c *DingTalkAITableConnector) loadRecordPage(ctx context.Context, sheetID, nextToken string, maxResults int) ([]dingTalkAITableRecord, string, error) {
+ if c.listRecords != nil {
+ return c.listRecords(ctx, sheetID, nextToken, maxResults)
+ }
+ values := url.Values{}
+ values.Set("operatorId", c.operatorID)
+ body := map[string]any{"maxResults": maxResults}
+ if nextToken != "" {
+ body["nextToken"] = nextToken
+ }
+ path := fmt.Sprintf("/v1.0/notable/bases/%s/sheets/%s/records/list?%s", url.PathEscape(c.tableID), url.PathEscape(sheetID), values.Encode())
+ var response dingTalkAITableRecordsResponse
+ if err := c.doJSON(ctx, http.MethodPost, path, body, &response); err != nil {
+ return nil, "", err
+ }
+ return response.Records, response.NextToken, nil
+}
+
+func (c *DingTalkAITableConnector) doJSON(ctx context.Context, method, path string, body any, out any) error {
+ if err := validateDingTalkAITableAPIBaseURL(c.apiBaseURL); err != nil {
+ return err
+ }
+ target := c.apiBaseURL + path
+ parsed, err := url.Parse(target)
+ if err != nil {
+ return err
+ }
+ if parsed.Scheme != "https" || parsed.Host != "api.dingtalk.com" {
+ return &ConnectorValidationError{Message: "DingTalk AI Table requests must target https://api.dingtalk.com"}
+ }
+ var reader io.Reader
+ if body != nil {
+ data, err := json.Marshal(body)
+ if err != nil {
+ return err
+ }
+ reader = bytes.NewReader(data)
+ }
+ req, err := http.NewRequestWithContext(ctx, method, target, reader)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("x-acs-dingtalk-access-token", c.accessToken)
+ req.Header.Set("Content-Type", "application/json")
+ resp, err := c.httpClient.Do(req)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ data, err := io.ReadAll(io.LimitReader(resp.Body, 32*1024*1024))
+ if err != nil {
+ return err
+ }
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return fmt.Errorf("DingTalk API returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(data)))
+ }
+ if len(data) == 0 || out == nil {
+ return nil
+ }
+ if err := json.Unmarshal(data, out); err != nil {
+ return fmt.Errorf("decode DingTalk API response: %w", err)
+ }
+ return nil
+}
+
+func validateDingTalkAITableAPIBaseURL(value string) error {
+ parsed, err := url.Parse(value)
+ if err != nil || parsed.Scheme != "https" || parsed.Host != "api.dingtalk.com" || parsed.Path != "" || parsed.RawQuery != "" || parsed.Fragment != "" {
+ return &ConnectorValidationError{Message: "DingTalk AI Table API base URL must be https://api.dingtalk.com"}
+ }
+ return nil
+}
+
+func (c *DingTalkAITableConnector) recordDocument(sheet dingTalkAITableSheet, record dingTalkAITableRecord) (SourceDocument, bool) {
+ blob, err := json.MarshalIndent(record.Fields, "", " ")
+ if err != nil {
+ blob = []byte("{}")
+ }
+ updatedAt, hasUpdatedAt := parseDingTalkAITableLastModifiedTime(record.LastModifiedTime)
+ semanticIdentifier := fmt.Sprintf("%s - Record %s", sheet.Name, record.ID)
+ fieldNames := make([]string, 0, len(record.Fields))
+ for name := range record.Fields {
+ fieldNames = append(fieldNames, name)
+ }
+ sort.Strings(fieldNames)
+ for _, name := range fieldNames {
+ value := record.Fields[name]
+ if text, ok := value.(string); ok {
+ text = strings.TrimSpace(text)
+ if text != "" && len([]rune(text)) < 100 {
+ semanticIdentifier = fmt.Sprintf("%s - %s", sheet.Name, truncateRunes(text, 50))
+ break
+ }
+ }
+ }
+ metadata := map[string]any{
+ "table_id": c.tableID,
+ "sheet_id": sheet.ID,
+ "sheet_name": sheet.Name,
+ "record_id": record.ID,
+ }
+ return SourceDocument{
+ SourceID: c.documentID(sheet.ID, record.ID),
+ SemanticIdentifier: semanticIdentifier,
+ Extension: ".json",
+ Blob: blob,
+ UpdatedAt: updatedAt,
+ SizeBytes: int64(len(blob)),
+ Metadata: metadata,
+ Fingerprint: dingTalkAITableFingerprint(blob),
+ }, hasUpdatedAt
+}
+
+func (c *DingTalkAITableConnector) documentID(sheetID, recordID string) string {
+ return dingTalkAITableDocIDPrefix + c.tableID + ":" + sheetID + ":" + recordID
+}
+
+func dingTalkAITableInWindow(updatedAt time.Time, request SyncRequest) bool {
+ if !request.FromBeginning && request.WindowStart != nil && updatedAt.Before(*request.WindowStart) {
+ return false
+ }
+ if !request.WindowEnd.IsZero() && updatedAt.After(request.WindowEnd) {
+ return false
+ }
+ return true
+}
+
+func dingTalkAITableRequiresTimestamp(request SyncRequest) bool {
+ return !request.FromBeginning && request.WindowStart != nil
+}
+
+func parseDingTalkAITableLastModifiedTime(value any) (time.Time, bool) {
+ var millis int64
+ switch typed := value.(type) {
+ case int:
+ millis = int64(typed)
+ case int64:
+ millis = typed
+ case float64:
+ millis = int64(typed)
+ case json.Number:
+ parsed, err := typed.Int64()
+ if err != nil {
+ return time.Time{}, false
+ }
+ millis = parsed
+ case string:
+ parsed, err := strconv.ParseInt(strings.TrimSpace(typed), 10, 64)
+ if err != nil {
+ return time.Time{}, false
+ }
+ millis = parsed
+ default:
+ return time.Time{}, false
+ }
+ if millis <= 0 {
+ return time.Time{}, false
+ }
+ return time.Unix(0, millis*int64(time.Millisecond)).UTC(), true
+}
+
+func dingTalkAITableFingerprint(blob []byte) string {
+ sum := xxh3.Hash128(blob).Bytes()
+ return hex.EncodeToString(sum[:])
+}
+
+func truncateRunes(value string, max int) string {
+ runes := []rune(value)
+ if len(runes) <= max {
+ return value
+ }
+ return string(runes[:max])
+}
+
+type dingTalkAITableSheet struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+}
+
+type dingTalkAITableRecord struct {
+ ID string `json:"id"`
+ Fields map[string]any `json:"fields"`
+ LastModifiedTime any `json:"lastModifiedTime"`
+}
+
+type dingTalkAITableSheetsResponse struct {
+ Value []dingTalkAITableSheet `json:"value"`
+}
+
+type dingTalkAITableRecordsResponse struct {
+ Records []dingTalkAITableRecord `json:"records"`
+ NextToken string `json:"nextToken"`
+}
+
+type dingTalkAITableSyncSession struct {
+ documents []SourceDocument
+ batchSize int
+ index int
+}
+
+func (s *dingTalkAITableSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ if s.index >= len(s.documents) {
+ return SyncBatch{}, io.EOF
+ }
+ end := s.index + s.batchSize
+ if end > len(s.documents) {
+ end = len(s.documents)
+ }
+ documents := s.documents[s.index:end]
+ batch := SyncBatch{Documents: documents, Checkpoint: dingTalkAITableCheckpoint(documents[len(documents)-1])}
+ s.index = end
+ return batch, nil
+}
+
+func (s *dingTalkAITableSyncSession) Close() error {
+ return nil
+}
+
+func (s *dingTalkAITableSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+ if checkpoint == nil {
+ return
+ }
+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ if sourceID == "" {
+ return
+ }
+ for index, doc := range s.documents {
+ if doc.SourceID == sourceID {
+ s.index = index + 1
+ return
+ }
+ }
+}
+
+func dingTalkAITableCheckpoint(doc SourceDocument) *SyncCheckpoint {
+ updatedAt := doc.UpdatedAt
+ return &SyncCheckpoint{Cursor: doc.SourceID, SourceID: doc.SourceID, UpdatedAt: &updatedAt}
+}
+
+type dingTalkAITablePruneSession struct {
+ documents []SlimDocument
+ batchSize int
+ index int
+}
+
+func (s *dingTalkAITablePruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ if s.index >= len(s.documents) {
+ return PruneBatch{}, io.EOF
+ }
+ end := s.index + s.batchSize
+ if end > len(s.documents) {
+ end = len(s.documents)
+ }
+ batch := PruneBatch{Documents: s.documents[s.index:end]}
+ s.index = end
+ return batch, nil
+}
+
+func (s *dingTalkAITablePruneSession) Close() error {
+ return nil
+}
diff --git a/internal/syncer/connector/dingtalk_ai_table_test.go b/internal/syncer/connector/dingtalk_ai_table_test.go
new file mode 100644
index 0000000000..080792c239
--- /dev/null
+++ b/internal/syncer/connector/dingtalk_ai_table_test.go
@@ -0,0 +1,277 @@
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "reflect"
+ "testing"
+ "time"
+)
+
+func TestDingTalkAITableConnectorOpenSync(t *testing.T) {
+ var recordRequests []map[string]any
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("x-acs-dingtalk-access-token") != "token-1" {
+ t.Fatalf("access token header = %q", r.Header.Get("x-acs-dingtalk-access-token"))
+ }
+ if r.URL.Query().Get("operatorId") != "operator-1" {
+ t.Fatalf("operatorId = %q", r.URL.Query().Get("operatorId"))
+ }
+ switch r.URL.Path {
+ case "/v1.0/notable/bases/table-1/sheets":
+ if r.Method != http.MethodGet {
+ t.Fatalf("sheets method = %s", r.Method)
+ }
+ writeJSON(t, w, map[string]any{
+ "value": []map[string]any{
+ {"id": "sheet-1", "name": "Projects"},
+ },
+ })
+ case "/v1.0/notable/bases/table-1/sheets/sheet-1/records/list":
+ if r.Method != http.MethodPost {
+ t.Fatalf("records method = %s", r.Method)
+ }
+ var body map[string]any
+ if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
+ t.Fatalf("decode request body: %v", err)
+ }
+ recordRequests = append(recordRequests, body)
+ if body["nextToken"] == nil {
+ writeJSON(t, w, map[string]any{
+ "nextToken": "page-2",
+ "records": []map[string]any{
+ {"id": "rec-1", "lastModifiedTime": int64(1780000000000), "fields": map[string]any{"Title": "Roadmap", "Count": 3}},
+ },
+ })
+ return
+ }
+ writeJSON(t, w, map[string]any{
+ "records": []map[string]any{
+ {"id": "rec-2", "lastModifiedTime": int64(1780000001000), "fields": map[string]any{"zTitle": "Wrong", "aTitle": "Budget"}},
+ },
+ })
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ connector, err := NewDingTalkAITableConnector(map[string]any{
+ "table_id": "table-1",
+ "operator_id": "operator-1",
+ "batch_size": 1,
+ "credentials": map[string]any{"access_token": "token-1"},
+ })
+ if err != nil {
+ t.Fatalf("NewDingTalkAITableConnector failed: %v", err)
+ }
+ connector.httpClient = dingTalkAITableTestHTTPClient(t, server)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ defer func() {
+ if err := session.Close(); err != nil {
+ t.Errorf("Close failed: %v", err)
+ }
+ }()
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch first failed: %v", err)
+ }
+ if len(first.Documents) != 1 {
+ t.Fatalf("first batch documents = %d, want 1", len(first.Documents))
+ }
+ doc := first.Documents[0]
+ if doc.SourceID != "dingtalk_ai_table:table-1:sheet-1:rec-1" {
+ t.Fatalf("SourceID = %q", doc.SourceID)
+ }
+ if doc.SemanticIdentifier != "Projects - Roadmap" {
+ t.Fatalf("SemanticIdentifier = %q", doc.SemanticIdentifier)
+ }
+ if got := doc.UpdatedAt; !got.Equal(time.UnixMilli(1780000000000).UTC()) {
+ t.Fatalf("UpdatedAt = %s, want %s", got, time.UnixMilli(1780000000000).UTC())
+ }
+ if doc.Extension != ".json" || doc.SizeBytes != int64(len(doc.Blob)) || doc.Fingerprint == "" {
+ t.Fatalf("document shape = ext %q size %d fingerprint %q", doc.Extension, doc.SizeBytes, doc.Fingerprint)
+ }
+ if got := doc.Metadata["record_id"]; got != "rec-1" {
+ t.Fatalf("record_id metadata = %v", got)
+ }
+ if first.Checkpoint == nil || first.Checkpoint.SourceID != doc.SourceID {
+ t.Fatalf("checkpoint = %#v", first.Checkpoint)
+ }
+
+ second, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch second failed: %v", err)
+ }
+ if got := second.Documents[0].SourceID; got != "dingtalk_ai_table:table-1:sheet-1:rec-2" {
+ t.Fatalf("second SourceID = %q", got)
+ }
+ if got := second.Documents[0].SemanticIdentifier; got != "Projects - Budget" {
+ t.Fatalf("second SemanticIdentifier = %q", got)
+ }
+ _, err = session.NextBatch(context.Background())
+ if !errors.Is(err, io.EOF) {
+ t.Fatalf("final error = %v, want EOF", err)
+ }
+ if len(recordRequests) != 2 || recordRequests[0]["maxResults"] != float64(100) || recordRequests[1]["nextToken"] != "page-2" {
+ t.Fatalf("record requests = %#v", recordRequests)
+ }
+}
+
+func TestDingTalkAITableConnectorOpenPrune(t *testing.T) {
+ connector := &DingTalkAITableConnector{
+ tableID: "table-1",
+ operatorID: "operator-1",
+ accessToken: "token-1",
+ batchSize: 10,
+ apiBaseURL: dingTalkAITableAPIBaseURL,
+ getSheets: func(ctx context.Context) ([]dingTalkAITableSheet, error) {
+ return []dingTalkAITableSheet{{ID: "sheet-1", Name: "Projects"}}, nil
+ },
+ listRecords: func(ctx context.Context, sheetID, nextToken string, maxResults int) ([]dingTalkAITableRecord, string, error) {
+ if nextToken == "" {
+ return []dingTalkAITableRecord{{ID: "rec-1"}, {ID: ""}}, "next", nil
+ }
+ return []dingTalkAITableRecord{{ID: "rec-2"}}, "", nil
+ },
+ }
+
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune failed: %v", err)
+ }
+ defer func() {
+ if err := session.Close(); err != nil {
+ t.Errorf("Close failed: %v", err)
+ }
+ }()
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ got := []string{batch.Documents[0].SourceID, batch.Documents[1].SourceID}
+ want := []string{
+ "dingtalk_ai_table:table-1:sheet-1:rec-1",
+ "dingtalk_ai_table:table-1:sheet-1:rec-2",
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Fatalf("slim ids = %#v, want %#v", got, want)
+ }
+}
+
+func TestDingTalkAITableConnectorOpenSyncFiltersByRecordLastModifiedTime(t *testing.T) {
+ windowStart := time.UnixMilli(1780000000000).UTC()
+ connector := &DingTalkAITableConnector{
+ tableID: "table-1",
+ operatorID: "operator-1",
+ accessToken: "token-1",
+ batchSize: 10,
+ apiBaseURL: dingTalkAITableAPIBaseURL,
+ getSheets: func(ctx context.Context) ([]dingTalkAITableSheet, error) {
+ return []dingTalkAITableSheet{{ID: "sheet-1", Name: "Projects"}}, nil
+ },
+ listRecords: func(ctx context.Context, sheetID, nextToken string, maxResults int) ([]dingTalkAITableRecord, string, error) {
+ return []dingTalkAITableRecord{
+ {ID: "old", LastModifiedTime: windowStart.Add(-time.Millisecond).UnixMilli(), Fields: map[string]any{"Title": "Old"}},
+ {ID: "inside", LastModifiedTime: windowStart.Add(time.Millisecond).UnixMilli(), Fields: map[string]any{"Title": "Inside"}},
+ {ID: "missing", Fields: map[string]any{"Title": "Missing"}},
+ {ID: "invalid", LastModifiedTime: "invalid", Fields: map[string]any{"Title": "Invalid"}},
+ }, "", nil
+ },
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &windowStart, WindowEnd: windowStart.Add(time.Hour)})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ defer func() {
+ if err := session.Close(); err != nil {
+ t.Errorf("Close failed: %v", err)
+ }
+ }()
+
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "dingtalk_ai_table:table-1:sheet-1:inside" {
+ t.Fatalf("documents = %#v, want only inside record", batch.Documents)
+ }
+}
+
+func TestDingTalkAITableConnectorValidateConnectorSetting(t *testing.T) {
+ config := map[string]any{
+ "table_id": "table-1",
+ "operator_id": "operator-1",
+ "credentials": map[string]any{},
+ }
+ connector, err := NewDingTalkAITableConnector(config)
+ if err != nil {
+ t.Fatalf("NewDingTalkAITableConnector failed: %v", err)
+ }
+
+ err = connector.ValidateConnectorSetting(context.Background(), config)
+ var credErr *ConnectorMissingCredentialError
+ if !errors.As(err, &credErr) {
+ t.Fatalf("error = %v, want ConnectorMissingCredentialError", err)
+ }
+}
+
+func TestDingTalkAITableConnectorRejectsUnapprovedAPIBaseURL(t *testing.T) {
+ connector := &DingTalkAITableConnector{
+ tableID: "table-1",
+ operatorID: "operator-1",
+ accessToken: "token-1",
+ batchSize: 10,
+ apiBaseURL: "http://example.com",
+ httpClient: http.DefaultClient,
+ }
+
+ err := connector.doJSON(context.Background(), http.MethodGet, "/v1.0/notable/bases/table-1/sheets", nil, nil)
+ var validationErr *ConnectorValidationError
+ if !errors.As(err, &validationErr) {
+ t.Fatalf("error = %v, want ConnectorValidationError", err)
+ }
+}
+
+func dingTalkAITableTestHTTPClient(t *testing.T, server *httptest.Server) *http.Client {
+ t.Helper()
+ target, err := url.Parse(server.URL)
+ if err != nil {
+ t.Fatalf("parse test server URL: %v", err)
+ }
+ return &http.Client{Transport: dingTalkAITableRoundTripFunc(func(req *http.Request) (*http.Response, error) {
+ if req.URL.Scheme != "https" || req.URL.Host != "api.dingtalk.com" {
+ t.Fatalf("request URL = %s, want https://api.dingtalk.com", req.URL.String())
+ }
+ cloned := req.Clone(req.Context())
+ cloned.URL.Scheme = target.Scheme
+ cloned.URL.Host = target.Host
+ cloned.Host = target.Host
+ return http.DefaultTransport.RoundTrip(cloned)
+ })}
+}
+
+type dingTalkAITableRoundTripFunc func(req *http.Request) (*http.Response, error)
+
+func (f dingTalkAITableRoundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+ return f(req)
+}
+
+func writeJSON(t *testing.T, w http.ResponseWriter, value any) {
+ t.Helper()
+ w.Header().Set("Content-Type", "application/json")
+ if err := json.NewEncoder(w).Encode(value); err != nil {
+ t.Fatalf("encode JSON: %v", err)
+ }
+}