RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Feat/connector go sharepoint (#18618)
mkaaad9fc323b49ac4
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 46a9ac9919..393698a006 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -47,6 +47,7 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "mysql", NewMySQLConnector)registerBuiltIn(registry, "postgresql", NewPostgreSQLConnector)registerBuiltIn(registry, "slack", NewSlackConnector)+ registerBuiltIn(registry, "sharepoint", NewSharePointConnector)registerBuiltIn(registry, "discord", NewDiscordConnector)registerBuiltIn(registry, "webdav", NewWebDAVConnector)}diff --git a/internal/syncer/connector/sharepoint.go b/internal/syncer/connector/sharepoint.gonew file mode 100644index 0000000000..34da15c154--- /dev/null+++ b/internal/syncer/connector/sharepoint.go@@ -0,0 +1,775 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "sort"+ "strings"+ "sync"+ "time"+)++// SharePoint connector constants.+const (+ sharepointGraphBase = "https://graph.microsoft.com/v1.0"+ sharepointTokenURLFormat = "https://login.microsoftonline.com/%s/oauth2/v2.0/token"+ sharepointGraphScope = "https://graph.microsoft.com/.default"+ sharepointDefaultBatchSize = 2+ sharepointRequestTimeout = 60 * time.Second+ sharepointRetryCount = 4+ sharepointRetryBaseDelay = 200 * time.Millisecond+ sharepointTokenExpiryMargin = 5 * time.Minute+ sharepointMaxJSONResponseSize = 16 * 1024 * 1024+ sharepointDefaultMaxFileSize = 256 * 1024 * 1024+)++// sharepointMaxFileSize caps downloaded file content. Package variable so+// tests can shrink it.+var sharepointMaxFileSize int64 = sharepointDefaultMaxFileSize++// SharePointConnector reads files from SharePoint document libraries through+// the Microsoft Graph API using an app-only (client-credentials) token.+type SharePointConnector struct {+ tenantID string+ clientID string+ clientSecret string+ siteURL string+ batchSize int++ clientMu sync.Mutex+ accessToken string+ tokenExpiry time.Time+ httpClient *http.Client+ now func() time.Time++ tokenURL string+ graphBaseURL string+}++// NewSharePointConnector creates a SharePoint connector from connector config.+func NewSharePointConnector(config map[string]any) (*SharePointConnector, error) {+ credentials := configAnyMap(config["credentials"])++ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), sharepointDefaultBatchSize)+ if batchSize <= 0 {+ batchSize = sharepointDefaultBatchSize+ }++ return &SharePointConnector{+ tenantID: strings.TrimSpace(stringConfig(credentials["tenant_id"])),+ clientID: strings.TrimSpace(stringConfig(credentials["client_id"])),+ clientSecret: stringConfig(credentials["client_secret"]),+ siteURL: strings.TrimSpace(stringConfig(credentials["site_url"])),+ batchSize: batchSize,+ httpClient: &http.Client{Timeout: sharepointRequestTimeout},+ tokenURL: "",+ graphBaseURL: sharepointGraphBase,+ }, nil+}++// Validate validates SharePoint connector settings, credentials, and access.+func (c *SharePointConnector) Validate(ctx context.Context) error {+ if c == nil {+ return &ConnectorValidationError{Message: "SharePoint connector is nil"}+ }+ if c.tenantID == "" || c.clientID == "" || c.clientSecret == "" || c.siteURL == "" {+ return &ConnectorMissingCredentialError{Message: "SharePoint credentials are incomplete: tenant_id, client_id, client_secret, and site_url are required"}+ }+ if c.batchSize <= 0 {+ return &ConnectorValidationError{Message: "SharePoint connector batch_size must be a positive integer"}+ }+ if _, err := c.token(ctx); err != nil {+ return &ConnectorMissingCredentialError{Message: fmt.Sprintf("Failed to acquire SharePoint access token: %v", err)}+ }+ if _, err := c.resolveSite(ctx); err != nil {+ return c.siteValidationError(err)+ }+ return nil+}++// ValidateConnectorSetting validates SharePoint settings from an unsaved config.+func (c *SharePointConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ return c.Validate(ctx)+}++// OpenSync opens one SharePoint sync session.+func (c *SharePointConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ site, err := c.resolveSite(ctx)+ if err != nil {+ return nil, err+ }+ drives, err := c.listDrives(ctx, site.ID)+ if err != nil {+ return nil, err+ }+ // Drives are processed in ID order so checkpoints advance deterministically.+ sort.SliceStable(drives, func(i, j int) bool {+ return drives[i].ID < drives[j].ID+ })+ session := &sharePointSyncSession{+ connector: c,+ request: request,+ site: site,+ drives: drives,+ batchSize: c.batchSize,+ }+ session.applyResume(request.Resume)+ return session, nil+}++// OpenPrune opens one complete SharePoint prune snapshot session.+func (c *SharePointConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ if err := c.Validate(ctx); err != nil {+ return nil, err+ }+ site, err := c.resolveSite(ctx)+ if err != nil {+ return nil, err+ }+ drives, err := c.listDrives(ctx, site.ID)+ if err != nil {+ return nil, err+ }+ var documents []SlimDocument+ for _, drive := range drives {+ files, err := c.walkDriveFiles(ctx, drive.ID)+ if err != nil {+ return nil, err+ }+ for _, file := range files {+ documents = append(documents, SlimDocument{SourceID: sharePointDocID(drive.ID, file.ID)})+ }+ }+ return &sharePointPruneSession{documents: documents, batchSize: c.batchSize}, nil+}++// ---------------------------------------------------------------------------+// Microsoft Graph client+// ---------------------------------------------------------------------------++type sharePointSite struct {+ ID string `json:"id"`+ Name string `json:"name"`+ WebURL string `json:"webUrl"`+}++type sharePointDrive struct {+ ID string `json:"id"`+ Name string `json:"name"`+ DriveType string `json:"driveType"`+}++type sharePointItem struct {+ ID string `json:"id"`+ Name string `json:"name"`+ Size int64 `json:"size"`+ WebURL string `json:"webUrl"`+ LastModifiedDateTime string `json:"lastModifiedDateTime"`+ Folder *sharePointFolderFacet `json:"folder,omitempty"`+ File *sharePointFileFacet `json:"file,omitempty"`+}++type sharePointFolderFacet struct {+ ChildCount int `json:"childCount"`+}++type sharePointFileFacet struct {+ MimeType string `json:"mimeType"`+}++type sharePointDrivesPage struct {+ NextLink string `json:"@odata.nextLink"`+ Value []sharePointDrive `json:"value"`+}++type sharePointChildrenPage struct {+ NextLink string `json:"@odata.nextLink"`+ Value []sharePointItem `json:"value"`+}++type sharePointTokenResponse struct {+ AccessToken string `json:"access_token"`+ ExpiresIn int `json:"expires_in"`+}++type sharePointCachedToken struct {+ accessToken string+ expiresAt time.Time+}++type sharePointGraphErrorBody struct {+ Error struct {+ Code string `json:"code"`+ Message string `json:"message"`+ } `json:"error"`+}++// sharePointAPIError marks a non-2xx Microsoft Graph response.+type sharePointAPIError struct {+ status int+ body string+}++func (e *sharePointAPIError) Error() string {+ return fmt.Sprintf("SharePoint Graph API request failed with status %d: %s", e.status, e.body)+}++// siteAPIPath converts a site URL such as+// https://contoso.sharepoint.com/sites/MySite into the Graph relative path+// /sites/contoso.sharepoint.com:/sites/MySite.+func (c *SharePointConnector) siteAPIPath() string {+ parsed, err := url.Parse(c.siteURL)+ if err != nil || parsed.Host == "" {+ return ""+ }+ pathPart := strings.Trim(parsed.Path, "/")+ if pathPart == "" {+ return "/sites/" + parsed.Host+ }+ segments := strings.Split(pathPart, "/")+ escaped := make([]string, len(segments))+ for i, segment := range segments {+ escaped[i] = url.PathEscape(segment)+ }+ return "/sites/" + parsed.Host + ":/" + strings.Join(escaped, "/")+}++// resolveSite resolves the configured site URL through the Graph API.+func (c *SharePointConnector) resolveSite(ctx context.Context) (*sharePointSite, error) {+ apiPath := c.siteAPIPath()+ if apiPath == "" {+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Invalid SharePoint site URL %q", c.siteURL)}+ }+ var site sharePointSite+ if err := c.getJSON(ctx, c.graphBaseURL+apiPath, &site); err != nil {+ return nil, err+ }+ if site.ID == "" {+ return nil, &ConnectorValidationError{Message: "Failed to access SharePoint site"}+ }+ return &site, nil+}++// listDrives lists every document library under the site, following+// @odata.nextLink pagination.+func (c *SharePointConnector) listDrives(ctx context.Context, siteID string) ([]sharePointDrive, error) {+ var drives []sharePointDrive+ apiURL := c.graphBaseURL + "/sites/" + url.PathEscape(siteID) + "/drives"+ for apiURL != "" {+ var page sharePointDrivesPage+ if err := c.getJSON(ctx, apiURL, &page); err != nil {+ return nil, err+ }+ drives = append(drives, page.Value...)+ apiURL = page.NextLink+ }+ return drives, nil+}++// listChildren lists the children of a drive item; an empty itemID refers to+// the drive root.+func (c *SharePointConnector) listChildren(ctx context.Context, driveID, itemID string) ([]sharePointItem, error) {+ var items []sharePointItem+ var apiURL string+ if itemID == "" {+ apiURL = c.graphBaseURL + "/drives/" + url.PathEscape(driveID) + "/root/children"+ } else {+ apiURL = c.graphBaseURL + "/drives/" + url.PathEscape(driveID) + "/items/" + url.PathEscape(itemID) + "/children"+ }+ for apiURL != "" {+ var page sharePointChildrenPage+ if err := c.getJSON(ctx, apiURL, &page); err != nil {+ return nil, err+ }+ items = append(items, page.Value...)+ apiURL = page.NextLink+ }+ return items, nil+}++// walkDriveFiles depth-first walks a drive yielding file (non-folder) items.+func (c *SharePointConnector) walkDriveFiles(ctx context.Context, driveID string) ([]sharePointItem, error) {+ var files []sharePointItem+ stack := []string{""}+ for len(stack) > 0 {+ folderID := stack[len(stack)-1]+ stack = stack[:len(stack)-1]+ children, err := c.listChildren(ctx, driveID, folderID)+ if err != nil {+ return nil, err+ }+ for _, child := range children {+ if child.Folder != nil {+ stack = append(stack, child.ID)+ } else {+ files = append(files, child)+ }+ }+ }+ return files, nil+}++// downloadContent downloads a file's raw bytes.+func (c *SharePointConnector) downloadContent(ctx context.Context, driveID, itemID string) ([]byte, error) {+ apiURL := c.graphBaseURL + "/drives/" + url.PathEscape(driveID) + "/items/" + url.PathEscape(itemID) + "/content"+ return c.getBytes(ctx, apiURL)+}++// getJSON fetches and decodes one Graph JSON response.+func (c *SharePointConnector) getJSON(ctx context.Context, apiURL string, out any) error {+ body, err := c.doGraphRequest(ctx, apiURL, sharepointMaxJSONResponseSize)+ if err != nil {+ return err+ }+ return json.Unmarshal(body, out)+}++// getBytes fetches a raw Graph response body (file content).+func (c *SharePointConnector) getBytes(ctx context.Context, apiURL string) ([]byte, error) {+ return c.doGraphRequest(ctx, apiURL, sharepointMaxFileSize)+}++// doGraphRequest issues one authorized Graph GET, retrying transient failures+// and refreshing an expired access token once on 401.+func (c *SharePointConnector) doGraphRequest(ctx context.Context, apiURL string, maxBody int64) ([]byte, error) {+ token, err := c.token(ctx)+ if err != nil {+ return nil, err+ }++ var lastErr error+ retriedUnauthorized := false+ for attempt := 1; attempt <= sharepointRetryCount; attempt++ {+ requestCtx, cancel := context.WithTimeout(ctx, sharepointRequestTimeout)+ req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, apiURL, nil)+ if err != nil {+ cancel()+ return nil, err+ }+ req.Header.Set("Authorization", "Bearer "+token)++ resp, err := c.httpClient.Do(req)+ if err != nil {+ cancel()+ lastErr = err+ } else {+ body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxBody+1))+ resp.Body.Close()+ cancel()+ if resp.StatusCode >= 400 {+ lastErr = sharePointGraphError(resp.StatusCode, body)+ if resp.StatusCode == http.StatusUnauthorized && !retriedUnauthorized {+ c.invalidateToken(token)+ token, err = c.token(ctx)+ if err != nil {+ return nil, err+ }+ retriedUnauthorized = true+ attempt--+ continue+ }+ if !sharePointRetryable(resp.StatusCode) {+ return nil, lastErr+ }+ } else if readErr != nil {+ return nil, readErr+ } else if int64(len(body)) > maxBody {+ return nil, fmt.Errorf("SharePoint API response from %s exceeds maximum size of %d bytes", apiURL, maxBody)+ } else {+ return body, nil+ }+ }+ if attempt == sharepointRetryCount {+ break+ }+ delay := time.Duration(attempt) * sharepointRetryBaseDelay+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case <-time.After(delay):+ }+ }+ return nil, lastErr+}++func sharePointGraphError(status int, body []byte) error {+ message := strings.TrimSpace(string(body))+ var parsed sharePointGraphErrorBody+ if json.Unmarshal(body, &parsed) == nil && parsed.Error.Message != "" {+ message = parsed.Error.Message+ }+ return &sharePointAPIError{status: status, body: message}+}++func sharePointRetryable(status int) bool {+ switch status {+ case http.StatusTooManyRequests,+ http.StatusInternalServerError,+ http.StatusBadGateway,+ http.StatusServiceUnavailable,+ http.StatusGatewayTimeout:+ return true+ }+ return false+}++// token returns a cached or freshly acquired app-only access token.+func (c *SharePointConnector) token(ctx context.Context) (string, error) {+ c.clientMu.Lock()+ if c.accessToken != "" && !c.cachedTokenExpiredLocked() {+ token := c.accessToken+ c.clientMu.Unlock()+ return token, nil+ }+ c.clientMu.Unlock()++ cached, err := c.requestAccessToken(ctx)+ if err != nil {+ return "", err+ }+ if cached.accessToken == "" {+ return "", &ConnectorMissingCredentialError{Message: "SharePoint token endpoint returned an empty access token"}+ }++ c.clientMu.Lock()+ c.accessToken = cached.accessToken+ c.tokenExpiry = cached.expiresAt+ c.clientMu.Unlock()+ return cached.accessToken, nil+}++func (c *SharePointConnector) cachedTokenExpiredLocked() bool {+ return c.tokenExpiry.IsZero() || !c.currentTime().Add(sharepointTokenExpiryMargin).Before(c.tokenExpiry)+}++func (c *SharePointConnector) invalidateToken(token string) {+ c.clientMu.Lock()+ defer c.clientMu.Unlock()+ if c.accessToken == token {+ c.accessToken = ""+ c.tokenExpiry = time.Time{}+ }+}++func (c *SharePointConnector) currentTime() time.Time {+ if c.now != nil {+ return c.now()+ }+ return time.Now()+}++func (c *SharePointConnector) requestAccessToken(ctx context.Context) (sharePointCachedToken, error) {+ tokenURL := c.tokenURL+ if tokenURL == "" {+ tokenURL = fmt.Sprintf(sharepointTokenURLFormat, url.PathEscape(c.tenantID))+ }+ form := url.Values{+ "client_id": {c.clientID},+ "client_secret": {c.clientSecret},+ "grant_type": {"client_credentials"},+ "scope": {sharepointGraphScope},+ }+ requestCtx, cancel := context.WithTimeout(ctx, sharepointRequestTimeout)+ defer cancel()+ req, err := http.NewRequestWithContext(requestCtx, http.MethodPost, tokenURL, strings.NewReader(form.Encode()))+ if err != nil {+ return sharePointCachedToken{}, err+ }+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")++ resp, err := c.httpClient.Do(req)+ if err != nil {+ return sharePointCachedToken{}, err+ }+ defer resp.Body.Close()+ const maxTokenResponseSize = 1 << 20+ body, err := io.ReadAll(io.LimitReader(resp.Body, maxTokenResponseSize+1))+ if err != nil {+ return sharePointCachedToken{}, err+ }+ if len(body) > maxTokenResponseSize {+ return sharePointCachedToken{}, fmt.Errorf("SharePoint token response exceeds maximum size of %d bytes", maxTokenResponseSize)+ }+ if resp.StatusCode >= 400 {+ message := strings.TrimSpace(string(body))+ var parsed sharePointGraphErrorBody+ if json.Unmarshal(body, &parsed) == nil && parsed.Error.Message != "" {+ message = parsed.Error.Message+ }+ return sharePointCachedToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Failed to acquire SharePoint access token: %s", message)}+ }+ var token sharePointTokenResponse+ if err := json.Unmarshal(body, &token); err != nil {+ return sharePointCachedToken{}, err+ }+ if token.ExpiresIn <= 0 {+ return sharePointCachedToken{}, fmt.Errorf("SharePoint token endpoint returned invalid expires_in")+ }+ return sharePointCachedToken{+ accessToken: token.AccessToken,+ expiresAt: c.currentTime().Add(time.Duration(token.ExpiresIn) * time.Second),+ }, nil+}++// siteValidationError maps Graph failures during settings validation.+func (c *SharePointConnector) siteValidationError(err error) error {+ var apiErr *sharePointAPIError+ if errors.As(err, &apiErr) {+ if apiErr.status == http.StatusUnauthorized || apiErr.status == http.StatusForbidden {+ return &ConnectorValidationError{Message: "Invalid credentials or insufficient permissions for SharePoint"}+ }+ }+ return &ConnectorValidationError{Message: fmt.Sprintf("SharePoint validation error: %v", err)}+}++// ---------------------------------------------------------------------------+// Document building+// ---------------------------------------------------------------------------++// sharePointDocID namespaces a Graph driveItem ID by its drive ID because+// driveItem IDs are only unique within a single drive.+func sharePointDocID(driveID, itemID string) string {+ return driveID + ":" + itemID+}++// sharePointExtension derives the file extension (with leading dot) from a+// file name, mirroring the connector's document shape.+func sharePointExtension(name string) string {+ if index := strings.LastIndex(name, "."); index >= 0 && index < len(name)-1 {+ return name[index:]+ }+ return ""+}++func (s *sharePointSyncSession) fileToDocument(ctx context.Context, drive sharePointDrive, file sharePointItem) (*SourceDocument, error) {+ name := file.Name+ if name == "" {+ name = file.ID+ }+ blob, err := s.connector.downloadContent(ctx, drive.ID, file.ID)+ if err != nil {+ return nil, err+ }++ sizeBytes := file.Size+ if sizeBytes <= 0 {+ sizeBytes = int64(len(blob))+ }++ modified := parseOutlookTime(file.LastModifiedDateTime)+ if modified.IsZero() {+ modified = time.Now().UTC()+ }++ metadata := map[string]any{+ "drive": drive.Name,+ "drive_id": drive.ID,+ "drive_item_id": file.ID,+ }+ if file.WebURL != "" {+ metadata["web_url"] = file.WebURL+ }++ return &SourceDocument{+ SourceID: sharePointDocID(drive.ID, file.ID),+ SemanticIdentifier: name,+ Extension: sharePointExtension(name),+ Blob: blob,+ UpdatedAt: modified,+ SizeBytes: sizeBytes,+ Metadata: metadata,+ }, nil+}++// ---------------------------------------------------------------------------+// Sync session with checkpoint resume+// ---------------------------------------------------------------------------++type sharePointSyncSession struct {+ connector *SharePointConnector+ request SyncRequest+ site *sharePointSite+ drives []sharePointDrive+ batchSize int++ resumeAfterDriveID string+ driveIndex int+ pending []SourceDocument+ pendingCheckpoint *SyncCheckpoint+ done bool+}++// NextBatch returns the next SharePoint document batch.+func (s *sharePointSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ for {+ if s.done {+ return SyncBatch{}, io.EOF+ }+ if len(s.pending) == 0 {+ if s.driveIndex >= len(s.drives) {+ s.done = true+ return SyncBatch{}, io.EOF+ }+ drive := s.drives[s.driveIndex]+ s.driveIndex+++ if drive.ID <= s.resumeAfterDriveID {+ continue+ }+ documents, err := s.driveDocuments(ctx, drive)+ if err != nil {+ return SyncBatch{}, err+ }+ if len(documents) == 0 {+ continue+ }+ s.pending = documents+ updatedAt := s.maxUpdatedAt(documents)+ checkpoint := &SyncCheckpoint{+ Cursor: fmt.Sprintf("sharepoint_drive_%s", drive.ID),+ SourceID: fmt.Sprintf("sharepoint_drive_%s", drive.ID),+ UpdatedAt: &updatedAt,+ }+ // The checkpoint only advances once the whole drive is flushed so+ // an interrupted run re-processes a partially committed drive.+ s.pendingCheckpoint = checkpoint+ }+ n := s.batchSize+ if n > len(s.pending) {+ n = len(s.pending)+ }+ chunk := s.pending[:n]+ s.pending = s.pending[n:]+ var checkpoint *SyncCheckpoint+ if len(s.pending) == 0 && s.pendingCheckpoint != nil {+ checkpoint = s.pendingCheckpoint+ s.pendingCheckpoint = nil+ }+ return SyncBatch{Documents: chunk, Checkpoint: checkpoint}, nil+ }+}++// Close closes the SharePoint sync session.+func (s *sharePointSyncSession) Close() error {+ return nil+}++func (s *sharePointSyncSession) applyResume(checkpoint *SyncCheckpoint) {+ if checkpoint == nil {+ return+ }+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)+ const prefix = "sharepoint_drive_"+ if !strings.HasPrefix(sourceID, prefix) {+ return+ }+ driveID := strings.TrimPrefix(sourceID, prefix)+ if driveID == "" {+ return+ }+ s.resumeAfterDriveID = driveID+}++func (s *sharePointSyncSession) maxUpdatedAt(documents []SourceDocument) time.Time {+ var max time.Time+ for _, document := range documents {+ if document.UpdatedAt.After(max) {+ max = document.UpdatedAt+ }+ }+ return max+}++// driveDocuments builds the documents for one drive, filtering files by the+// sync window before downloading content.+func (s *sharePointSyncSession) driveDocuments(ctx context.Context, drive sharePointDrive) ([]SourceDocument, error) {+ files, err := s.connector.walkDriveFiles(ctx, drive.ID)+ if err != nil {+ return nil, err+ }+ var documents []SourceDocument+ for _, file := range files {+ if !s.fileInWindow(file) {+ continue+ }+ doc, err := s.fileToDocument(ctx, drive, file)+ if err != nil {+ return nil, err+ }+ if doc != nil {+ documents = append(documents, *doc)+ }+ }+ return documents, nil+}++// fileInWindow reports whether a file's lastModifiedDateTime falls in the+// exclusive-open, inclusive-closed sync window. Files without a parseable+// modification time are always included.+func (s *sharePointSyncSession) fileInWindow(file sharePointItem) bool {+ modified := parseOutlookTime(file.LastModifiedDateTime)+ if modified.IsZero() {+ return true+ }+ if s.request.WindowStart != nil && !modified.After(*s.request.WindowStart) {+ return false+ }+ if !s.request.WindowEnd.IsZero() && modified.After(s.request.WindowEnd) {+ return false+ }+ return true+}++// ---------------------------------------------------------------------------+// Prune session+// ---------------------------------------------------------------------------++type sharePointPruneSession struct {+ documents []SlimDocument+ batchSize int+ offset int+}++// NextBatch returns the next slim SharePoint snapshot batch.+func (s *sharePointPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ if s.offset >= len(s.documents) {+ return PruneBatch{}, io.EOF+ }+ n := s.batchSize+ if n <= 0 || n > len(s.documents)-s.offset {+ n = len(s.documents) - s.offset+ }+ batch := s.documents[s.offset : s.offset+n]+ s.offset += n+ return PruneBatch{Documents: batch}, nil+}++// Close closes the SharePoint prune session.+func (s *sharePointPruneSession) Close() error {+ return nil+}diff --git a/internal/syncer/connector/sharepoint_test.go b/internal/syncer/connector/sharepoint_test.gonew file mode 100644index 0000000000..05ee017b90--- /dev/null+++ b/internal/syncer/connector/sharepoint_test.go@@ -0,0 +1,682 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "strconv"+ "strings"+ "sync/atomic"+ "testing"+ "time"+)++func mustSharePointConnector(t *testing.T, serverURL string, extra map[string]any) *SharePointConnector {+ t.Helper()+ config := map[string]any{+ "batch_size": 2,+ "credentials": map[string]any{+ "tenant_id": "tenant",+ "client_id": "client",+ "client_secret": "secret",+ "site_url": "https://contoso.sharepoint.com/sites/MySite",+ },+ }+ for key, value := range extra {+ config[key] = value+ }+ connector, err := NewSharePointConnector(config)+ if err != nil {+ t.Fatalf("NewSharePointConnector: %v", err)+ }+ if serverURL != "" {+ connector.tokenURL = serverURL + "/oauth2/v2.0/token"+ connector.graphBaseURL = serverURL + "/v1.0"+ }+ return connector+}++type sharePointTestFixtures struct {+ tokenStatus int+ tokenBody string++ siteStatus int+ siteBody string+ siteCalls *atomic.Int64++ drivesBody string+ drivesBody2 string+ drivesCalls *atomic.Int64++ childrenByKey map[string]string+ childrenPage2 map[string]string+ childrenCalls *atomic.Int64++ contentByKey map[string]string+ contentCalls *atomic.Int64++ tokenCalls *atomic.Int64+}++const sharePointDefaultSiteBody = `{"id":"site-1","name":"MySite","webUrl":"https://contoso.sharepoint.com/sites/MySite"}`++func newTestSharePointServer(t *testing.T, fixtures *sharePointTestFixtures) *httptest.Server {+ t.Helper()+ if fixtures.tokenStatus == 0 {+ fixtures.tokenStatus = http.StatusOK+ }+ if fixtures.tokenBody == "" {+ fixtures.tokenBody = `{"access_token":"tok-1","expires_in":3600}`+ }+ if fixtures.siteStatus == 0 {+ fixtures.siteStatus = http.StatusOK+ }+ if fixtures.siteBody == "" {+ fixtures.siteBody = sharePointDefaultSiteBody+ }+ if fixtures.drivesBody == "" {+ fixtures.drivesBody = `{"value":[]}`+ }++ mux := http.NewServeMux()+ server := httptest.NewServer(mux)+ t.Cleanup(server.Close)++ mux.HandleFunc("/oauth2/v2.0/token", func(w http.ResponseWriter, r *http.Request) {+ if fixtures.tokenCalls != nil {+ fixtures.tokenCalls.Add(1)+ }+ w.WriteHeader(fixtures.tokenStatus)+ w.Write([]byte(fixtures.tokenBody))+ })++ mux.HandleFunc("/v1.0/", func(w http.ResponseWriter, r *http.Request) {+ rest := strings.TrimPrefix(r.URL.Path, "/v1.0/")+ switch {+ case strings.HasPrefix(rest, "sites/"):+ if strings.HasSuffix(rest, "/drives") {+ if fixtures.drivesCalls != nil {+ fixtures.drivesCalls.Add(1)+ }+ if r.URL.Query().Get("page") == "2" && fixtures.drivesBody2 != "" {+ w.Write([]byte(fixtures.drivesBody2))+ return+ }+ w.Write([]byte(fixtures.drivesBody))+ return+ }+ if fixtures.siteCalls != nil {+ fixtures.siteCalls.Add(1)+ }+ w.WriteHeader(fixtures.siteStatus)+ w.Write([]byte(fixtures.siteBody))+ case strings.HasPrefix(rest, "drives/"):+ parts := strings.Split(rest, "/")+ if len(parts) < 4 {+ http.NotFound(w, r)+ return+ }+ driveID := parts[1]+ switch parts[2] {+ case "root":+ if len(parts) != 4 || parts[3] != "children" {+ http.NotFound(w, r)+ return+ }+ if fixtures.childrenCalls != nil {+ fixtures.childrenCalls.Add(1)+ }+ key := driveID + ":/"+ if r.URL.Query().Get("page") == "2" && fixtures.childrenPage2[key] != "" {+ w.Write([]byte(fixtures.childrenPage2[key]))+ return+ }+ w.Write([]byte(fixtures.childrenByKey[key]))+ case "items":+ if len(parts) != 5 {+ http.NotFound(w, r)+ return+ }+ itemID := parts[3]+ switch parts[4] {+ case "children":+ if fixtures.childrenCalls != nil {+ fixtures.childrenCalls.Add(1)+ }+ key := driveID + ":" + itemID+ if r.URL.Query().Get("page") == "2" && fixtures.childrenPage2[key] != "" {+ w.Write([]byte(fixtures.childrenPage2[key]))+ return+ }+ w.Write([]byte(fixtures.childrenByKey[key]))+ case "content":+ if fixtures.contentCalls != nil {+ fixtures.contentCalls.Add(1)+ }+ w.Write([]byte(fixtures.contentByKey[driveID+":"+itemID]))+ default:+ http.NotFound(w, r)+ }+ default:+ http.NotFound(w, r)+ }+ default:+ http.NotFound(w, r)+ }+ })+ return server+}++func drainSharePointSync(t *testing.T, session SyncSession) []SourceDocument {+ t.Helper()+ var documents []SourceDocument+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ return documents+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ documents = append(documents, batch.Documents...)+ }+}++func sharePointFileJSON(itemID, name, webURL, modified string, size int64) string {+ return `{"id":"` + itemID + `","name":"` + name + `","webUrl":"` + webURL ++ `","lastModifiedDateTime":"` + modified + `","size":` + strconv.FormatInt(size, 10) ++ `,"file":{"mimeType":"text/plain"}}`+}++func sharePointFolderJSON(itemID, name string) string {+ return `{"id":"` + itemID + `","name":"` + name + `","folder":{"childCount":1}}`+}++func TestSharePointValidateMissingCredentials(t *testing.T) {+ connector := mustSharePointConnector(t, "", map[string]any{+ "credentials": map[string]any{"tenant_id": "t", "client_id": "c"},+ })+ err := connector.Validate(context.Background())+ if err == nil {+ t.Fatal("Validate succeeded, want error")+ }+ var missing *ConnectorMissingCredentialError+ if !errors.As(err, &missing) {+ t.Fatalf("error type = %T, want *ConnectorMissingCredentialError", err)+ }+}++func TestSharePointValidateSuccess(t *testing.T) {+ fixtures := &sharePointTestFixtures{+ tokenCalls: &atomic.Int64{},+ siteCalls: &atomic.Int64{},+ }+ server := newTestSharePointServer(t, fixtures)+ connector := mustSharePointConnector(t, server.URL, nil)++ if err := connector.Validate(context.Background()); err != nil {+ t.Fatalf("Validate: %v", err)+ }+ if fixtures.tokenCalls.Load() != 1 {+ t.Fatalf("token calls = %d, want 1", fixtures.tokenCalls.Load())+ }+ if fixtures.siteCalls.Load() != 1 {+ t.Fatalf("site calls = %d, want 1", fixtures.siteCalls.Load())+ }+}++func TestSharePointValidateTokenFailure(t *testing.T) {+ fixtures := &sharePointTestFixtures{+ tokenStatus: http.StatusUnauthorized,+ tokenBody: `{"error":"invalid_client","error_description":"bad secret"}`,+ }+ server := newTestSharePointServer(t, fixtures)+ connector := mustSharePointConnector(t, server.URL, nil)++ err := connector.Validate(context.Background())+ if err == nil {+ t.Fatal("Validate succeeded, want error")+ }+ var missing *ConnectorMissingCredentialError+ if !errors.As(err, &missing) {+ t.Fatalf("error type = %T, want *ConnectorMissingCredentialError", err)+ }+ if !strings.Contains(err.Error(), "Failed to acquire SharePoint access token") {+ t.Fatalf("error = %q, want token acquisition failure", err)+ }+}++func TestSharePointValidateSiteForbidden(t *testing.T) {+ fixtures := &sharePointTestFixtures{+ siteStatus: http.StatusForbidden,+ siteBody: `{"error":{"code":"accessDenied","message":"Access denied"}}`,+ }+ server := newTestSharePointServer(t, fixtures)+ connector := mustSharePointConnector(t, server.URL, nil)++ err := connector.Validate(context.Background())+ if err == nil {+ t.Fatal("Validate succeeded, want error")+ }+ var validation *ConnectorValidationError+ if !errors.As(err, &validation) {+ t.Fatalf("error type = %T, want *ConnectorValidationError", err)+ }+ if !strings.Contains(err.Error(), "Invalid credentials or insufficient permissions") {+ t.Fatalf("error = %q, want permission message", err)+ }+}++func TestSharePointValidateSiteNotFound(t *testing.T) {+ fixtures := &sharePointTestFixtures{+ siteStatus: http.StatusNotFound,+ siteBody: `{"error":{"code":"itemNotFound","message":"Site not found"}}`,+ }+ server := newTestSharePointServer(t, fixtures)+ connector := mustSharePointConnector(t, server.URL, nil)++ err := connector.Validate(context.Background())+ if err == nil {+ t.Fatal("Validate succeeded, want error")+ }+ var validation *ConnectorValidationError+ if !errors.As(err, &validation) {+ t.Fatalf("error type = %T, want *ConnectorValidationError", err)+ }+ if !strings.Contains(err.Error(), "SharePoint validation error") {+ t.Fatalf("error = %q, want validation error message", err)+ }+}++func TestSharePointSiteAPIPath(t *testing.T) {+ connector := mustSharePointConnector(t, "", map[string]any{+ "credentials": map[string]any{+ "tenant_id": "t", "client_id": "c", "client_secret": "s",+ "site_url": "https://contoso.sharepoint.com/sites/MySite",+ },+ })+ if got := connector.siteAPIPath(); got != "/sites/contoso.sharepoint.com:/sites/MySite" {+ t.Fatalf("siteAPIPath() = %q", got)+ }++ connector.siteURL = "https://contoso.sharepoint.com/sites/MySite/"+ if got := connector.siteAPIPath(); got != "/sites/contoso.sharepoint.com:/sites/MySite" {+ t.Fatalf("siteAPIPath(trailing slash) = %q", got)+ }++ connector.siteURL = "https://contoso.sharepoint.com/"+ if got := connector.siteAPIPath(); got != "/sites/contoso.sharepoint.com" {+ t.Fatalf("siteAPIPath(root) = %q", got)+ }++ connector.siteURL = "not a url"+ if got := connector.siteAPIPath(); got != "" {+ t.Fatalf("siteAPIPath(invalid) = %q, want empty", got)+ }+}++func TestSharePointOpenSyncWalksLibrariesAndDownloads(t *testing.T) {+ fixtures := &sharePointTestFixtures{+ drivesBody: `{"value":[+ {"id":"drv-A","name":"Documents","driveType":"documentLibrary"},+ {"id":"drv-B","name":"SiteAssets","driveType":"documentLibrary"}+ ]}`,+ childrenByKey: map[string]string{+ "drv-A:/": `{"value":[+ ` + sharePointFileJSON("f1", "readme.txt", "https://contoso.sharepoint.com/f1", "2026-01-01T12:00:00Z", 16) + `,+ ` + sharePointFolderJSON("d2", "sub") + `+ ]}`,+ "drv-A:d2": `{"value":[` + sharePointFileJSON("f2", "report.md", "https://contoso.sharepoint.com/f2", "2026-02-01T12:00:00Z", 8) + `]}`,+ "drv-B:/": `{"value":[` + sharePointFileJSON("f3", "logo.png", "https://contoso.sharepoint.com/f3", "2026-03-01T12:00:00Z", 5) + `]}`,+ },+ contentByKey: map[string]string{+ "drv-A:f1": "hello sharepoint",+ "drv-A:f2": "# Report",+ "drv-B:f3": "PNGDATA",+ },+ contentCalls: &atomic.Int64{},+ }+ server := newTestSharePointServer(t, fixtures)+ connector := mustSharePointConnector(t, server.URL, nil)++ session, err := connector.OpenSync(context.Background(), SyncRequest{})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ documents := drainSharePointSync(t, session)++ if len(documents) != 3 {+ t.Fatalf("documents = %d, want 3", len(documents))+ }+ byID := map[string]SourceDocument{}+ for _, document := range documents {+ byID[document.SourceID] = document+ }+ if len(byID) != 3 {+ t.Fatalf("duplicate SourceIDs: %v", documents)+ }+ for _, wantID := range []string{"drv-A:f1", "drv-A:f2", "drv-B:f3"} {+ if _, ok := byID[wantID]; !ok {+ t.Fatalf("missing document %q in %v", wantID, byID)+ }+ }++ first := byID["drv-A:f1"]+ if string(first.Blob) != "hello sharepoint" {+ t.Fatalf("f1 blob = %q", first.Blob)+ }+ if first.SemanticIdentifier != "readme.txt" {+ t.Fatalf("f1 semantic = %q", first.SemanticIdentifier)+ }+ if first.Extension != ".txt" {+ t.Fatalf("f1 extension = %q", first.Extension)+ }+ if first.SizeBytes != 16 {+ t.Fatalf("f1 size = %d, want 16", first.SizeBytes)+ }+ if first.Metadata["drive"] != "Documents" || first.Metadata["drive_id"] != "drv-A" || first.Metadata["drive_item_id"] != "f1" {+ t.Fatalf("f1 metadata = %v", first.Metadata)+ }+ if first.Metadata["web_url"] != "https://contoso.sharepoint.com/f1" {+ t.Fatalf("f1 web_url = %v", first.Metadata["web_url"])+ }+ wantModified := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)+ if !first.UpdatedAt.Equal(wantModified) {+ t.Fatalf("f1 updated_at = %v, want %v", first.UpdatedAt, wantModified)+ }+ if byID["drv-A:f2"].Extension != ".md" {+ t.Fatalf("f2 extension = %q", byID["drv-A:f2"].Extension)+ }+ if fixtures.contentCalls.Load() != 3 {+ t.Fatalf("content calls = %d, want 3", fixtures.contentCalls.Load())+ }+}++func TestSharePointWindowFilter(t *testing.T) {+ fixtures := &sharePointTestFixtures{+ drivesBody: `{"value":[{"id":"drv-A","name":"Documents","driveType":"documentLibrary"}]}`,+ childrenByKey: map[string]string{+ "drv-A:/": `{"value":[+ ` + sharePointFileJSON("f1", "jan.txt", "https://contoso.sharepoint.com/f1", "2026-01-01T12:00:00Z", 1) + `,+ ` + sharePointFileJSON("f2", "feb.txt", "https://contoso.sharepoint.com/f2", "2026-02-01T12:00:00Z", 1) + `,+ {"id":"f3","name":"no-time.txt","webUrl":"https://contoso.sharepoint.com/f3","size":1,"file":{"mimeType":"text/plain"}}+ ]}`,+ },+ contentByKey: map[string]string{+ "drv-A:f1": "a", "drv-A:f2": "b", "drv-A:f3": "c",+ },+ }+ server := newTestSharePointServer(t, fixtures)+ connector := mustSharePointConnector(t, server.URL, nil)++ windowStart := time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC)+ windowEnd := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)+ session, err := connector.OpenSync(context.Background(), SyncRequest{+ WindowStart: &windowStart,+ WindowEnd: windowEnd,+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ documents := drainSharePointSync(t, session)+ if len(documents) != 2 {+ t.Fatalf("incremental documents = %d, want 2 (f2, f3)", len(documents))+ }+ ids := map[string]bool{}+ for _, document := range documents {+ ids[document.SourceID] = true+ }+ if !ids["drv-A:f2"] || !ids["drv-A:f3"] || ids["drv-A:f1"] {+ t.Fatalf("incremental ids = %v", ids)+ }++ fullSession, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync(full): %v", err)+ }+ full := drainSharePointSync(t, fullSession)+ if len(full) != 3 {+ t.Fatalf("full documents = %d, want 3", len(full))+ }+}++func TestSharePointResumeSkipsSyncedDrives(t *testing.T) {+ fixtures := &sharePointTestFixtures{+ drivesBody: `{"value":[+ {"id":"drv-A","name":"A","driveType":"documentLibrary"},+ {"id":"drv-B","name":"B","driveType":"documentLibrary"},+ {"id":"drv-C","name":"C","driveType":"documentLibrary"}+ ]}`,+ childrenByKey: map[string]string{+ "drv-A:/": `{"value":[` + sharePointFileJSON("a1", "a.txt", "https://c/a1", "2026-01-01T00:00:00Z", 1) + `]}`,+ "drv-B:/": `{"value":[` + sharePointFileJSON("b1", "b.txt", "https://c/b1", "2026-01-01T00:00:00Z", 1) + `]}`,+ "drv-C:/": `{"value":[` + sharePointFileJSON("c1", "c.txt", "https://c/c1", "2026-01-01T00:00:00Z", 1) + `]}`,+ },+ contentByKey: map[string]string{+ "drv-A:a1": "a", "drv-B:b1": "b", "drv-C:c1": "c",+ },+ contentCalls: &atomic.Int64{},+ }+ server := newTestSharePointServer(t, fixtures)+ connector := mustSharePointConnector(t, server.URL, nil)++ session, err := connector.OpenSync(context.Background(), SyncRequest{+ Resume: &SyncCheckpoint{Cursor: "sharepoint_drive_drv-A"},+ })+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }++ var checkpoints []*SyncCheckpoint+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ if batch.Checkpoint != nil {+ checkpoints = append(checkpoints, batch.Checkpoint)+ }+ }+ if len(checkpoints) != 2 {+ t.Fatalf("checkpoints = %d, want 2", len(checkpoints))+ }+ if checkpoints[0].Cursor != "sharepoint_drive_drv-B" || checkpoints[1].Cursor != "sharepoint_drive_drv-C" {+ t.Fatalf("checkpoint cursors = %q, %q", checkpoints[0].Cursor, checkpoints[1].Cursor)+ }+ if fixtures.contentCalls.Load() != 2 {+ t.Fatalf("content calls = %d, want 2 (drv-A skipped)", fixtures.contentCalls.Load())+ }+}++func TestSharePointCheckpointAdvancesAfterDriveFlush(t *testing.T) {+ fixtures := &sharePointTestFixtures{+ drivesBody: `{"value":[+ {"id":"drv-A","name":"A","driveType":"documentLibrary"},+ {"id":"drv-B","name":"B","driveType":"documentLibrary"}+ ]}`,+ childrenByKey: map[string]string{+ "drv-A:/": `{"value":[` + sharePointFileJSON("a1", "a.txt", "https://c/a1", "2026-01-01T00:00:00Z", 1) + `]}`,+ "drv-B:/": `{"value":[` + sharePointFileJSON("b1", "b.txt", "https://c/b1", "2026-01-01T00:00:00Z", 1) + `]}`,+ },+ contentByKey: map[string]string{"drv-A:a1": "a", "drv-B:b1": "b"},+ }+ server := newTestSharePointServer(t, fixtures)+ connector := mustSharePointConnector(t, server.URL, nil)++ session, err := connector.OpenSync(context.Background(), SyncRequest{})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }++ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch: %v", err)+ }+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "drv-A:a1" {+ t.Fatalf("first batch = %+v", first)+ }+ if first.Checkpoint == nil || first.Checkpoint.Cursor != "sharepoint_drive_drv-A" {+ t.Fatalf("first checkpoint = %+v, want sharepoint_drive_drv-A", first.Checkpoint)+ }++ second, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("second NextBatch: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "drv-B:b1" {+ t.Fatalf("second batch = %+v", second)+ }+ if second.Checkpoint == nil || second.Checkpoint.Cursor != "sharepoint_drive_drv-B" {+ t.Fatalf("second checkpoint = %+v, want sharepoint_drive_drv-B", second.Checkpoint)+ }++ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("final NextBatch err = %v, want io.EOF", err)+ }+}++func TestSharePointOpenPruneListsIdsWithoutDownload(t *testing.T) {+ fixtures := &sharePointTestFixtures{+ drivesBody: `{"value":[+ {"id":"drv-A","name":"Documents","driveType":"documentLibrary"},+ {"id":"drv-B","name":"SiteAssets","driveType":"documentLibrary"}+ ]}`,+ childrenByKey: map[string]string{+ "drv-A:/": `{"value":[+ ` + sharePointFileJSON("f1", "a.txt", "https://c/f1", "2026-01-01T00:00:00Z", 1) + `,+ ` + sharePointFolderJSON("d2", "sub") + `+ ]}`,+ "drv-A:d2": `{"value":[` + sharePointFileJSON("f2", "b.txt", "https://c/f2", "2026-01-01T00:00:00Z", 1) + `]}`,+ "drv-B:/": `{"value":[` + sharePointFileJSON("f3", "c.txt", "https://c/f3", "2026-01-01T00:00:00Z", 1) + `]}`,+ },+ contentCalls: &atomic.Int64{},+ }+ server := newTestSharePointServer(t, fixtures)+ connector := mustSharePointConnector(t, server.URL, nil)++ prune, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune: %v", err)+ }+ var ids []string+ for {+ batch, err := prune.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch: %v", err)+ }+ for _, document := range batch.Documents {+ ids = append(ids, document.SourceID)+ }+ }+ if len(ids) != 3 {+ t.Fatalf("slim ids = %v, want 3", ids)+ }+ seen := map[string]bool{}+ for _, id := range ids {+ seen[id] = true+ }+ for _, want := range []string{"drv-A:f1", "drv-A:f2", "drv-B:f3"} {+ if !seen[want] {+ t.Fatalf("missing slim id %q in %v", want, ids)+ }+ }+ if fixtures.contentCalls.Load() != 0 {+ t.Fatalf("content calls = %d, want 0", fixtures.contentCalls.Load())+ }+}++func TestSharePointChildrenPagination(t *testing.T) {+ fixtures := &sharePointTestFixtures{+ drivesBody: `{"value":[{"id":"drv-A","name":"Documents","driveType":"documentLibrary"}]}`,+ childrenPage2: map[string]string{+ "drv-A:/": `{"value":[` + sharePointFileJSON("f2", "two.txt", "https://c/f2", "2026-01-01T00:00:00Z", 1) + `]}`,+ },+ contentByKey: map[string]string{"drv-A:f1": "one", "drv-A:f2": "two"},+ }+ server := newTestSharePointServer(t, fixtures)+ fixtures.childrenByKey = map[string]string{+ "drv-A:/": `{"@odata.nextLink":"` + server.URL + `/v1.0/drives/drv-A/root/children?page=2","value":[` ++ sharePointFileJSON("f1", "one.txt", "https://c/f1", "2026-01-01T00:00:00Z", 1) + `]}`,+ }+ connector := mustSharePointConnector(t, server.URL, nil)++ session, err := connector.OpenSync(context.Background(), SyncRequest{})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ documents := drainSharePointSync(t, session)+ if len(documents) != 2 {+ t.Fatalf("documents = %d, want 2 (paginated)", len(documents))+ }+}++func TestSharePointTokenCachedAcrossCalls(t *testing.T) {+ fixtures := &sharePointTestFixtures{+ tokenCalls: &atomic.Int64{},+ drivesBody: `{"value":[{"id":"drv-A","name":"Documents","driveType":"documentLibrary"}]}`,+ childrenByKey: map[string]string{+ "drv-A:/": `{"value":[` + sharePointFileJSON("f1", "a.txt", "https://c/f1", "2026-01-01T00:00:00Z", 1) + `]}`,+ },+ contentByKey: map[string]string{"drv-A:f1": "a"},+ }+ server := newTestSharePointServer(t, fixtures)+ connector := mustSharePointConnector(t, server.URL, nil)++ session, err := connector.OpenSync(context.Background(), SyncRequest{})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ drainSharePointSync(t, session)++ if fixtures.tokenCalls.Load() != 1 {+ t.Fatalf("token calls = %d, want 1 (cached)", fixtures.tokenCalls.Load())+ }+}++func TestSharePointFileSizeCap(t *testing.T) {+ origMax := sharepointMaxFileSize+ sharepointMaxFileSize = 4+ t.Cleanup(func() { sharepointMaxFileSize = origMax })++ fixtures := &sharePointTestFixtures{+ drivesBody: `{"value":[{"id":"drv-A","name":"Documents","driveType":"documentLibrary"}]}`,+ childrenByKey: map[string]string{+ "drv-A:/": `{"value":[` + sharePointFileJSON("f1", "big.txt", "https://c/f1", "2026-01-01T00:00:00Z", 100) + `]}`,+ },+ contentByKey: map[string]string{"drv-A:f1": "0123456789"},+ }+ server := newTestSharePointServer(t, fixtures)+ connector := mustSharePointConnector(t, server.URL, nil)++ session, err := connector.OpenSync(context.Background(), SyncRequest{})+ if err != nil {+ t.Fatalf("OpenSync: %v", err)+ }+ if _, err := session.NextBatch(context.Background()); err == nil || !strings.Contains(err.Error(), "exceeds maximum size") {+ t.Fatalf("NextBatch err = %v, want size limit error", err)+ }+}