Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

Feat/connector go sharepoint (#18618)

mkaaad9fc323b49ac4
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index 46a9ac9919..393698a006 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -47,6 +47,7 @@ func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "mysql", NewMySQLConnector)
registerBuiltIn(registry, "postgresql", NewPostgreSQLConnector)
registerBuiltIn(registry, "slack", NewSlackConnector)
+ registerBuiltIn(registry, "sharepoint", NewSharePointConnector)
registerBuiltIn(registry, "discord", NewDiscordConnector)
registerBuiltIn(registry, "webdav", NewWebDAVConnector)
}
diff --git a/internal/syncer/connector/sharepoint.go b/internal/syncer/connector/sharepoint.go
new file mode 100644
index 0000000000..34da15c154
--- /dev/null
+++ b/internal/syncer/connector/sharepoint.go
@@ -0,0 +1,775 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "sort"
+ "strings"
+ "sync"
+ "time"
+)
+
+// SharePoint connector constants.
+const (
+ sharepointGraphBase = "https://graph.microsoft.com/v1.0"
+ sharepointTokenURLFormat = "https://login.microsoftonline.com/%s/oauth2/v2.0/token"
+ sharepointGraphScope = "https://graph.microsoft.com/.default"
+ sharepointDefaultBatchSize = 2
+ sharepointRequestTimeout = 60 * time.Second
+ sharepointRetryCount = 4
+ sharepointRetryBaseDelay = 200 * time.Millisecond
+ sharepointTokenExpiryMargin = 5 * time.Minute
+ sharepointMaxJSONResponseSize = 16 * 1024 * 1024
+ sharepointDefaultMaxFileSize = 256 * 1024 * 1024
+)
+
+// sharepointMaxFileSize caps downloaded file content. Package variable so
+// tests can shrink it.
+var sharepointMaxFileSize int64 = sharepointDefaultMaxFileSize
+
+// SharePointConnector reads files from SharePoint document libraries through
+// the Microsoft Graph API using an app-only (client-credentials) token.
+type SharePointConnector struct {
+ tenantID string
+ clientID string
+ clientSecret string
+ siteURL string
+ batchSize int
+
+ clientMu sync.Mutex
+ accessToken string
+ tokenExpiry time.Time
+ httpClient *http.Client
+ now func() time.Time
+
+ tokenURL string
+ graphBaseURL string
+}
+
+// NewSharePointConnector creates a SharePoint connector from connector config.
+func NewSharePointConnector(config map[string]any) (*SharePointConnector, error) {
+ credentials := configAnyMap(config["credentials"])
+
+ batchSize := configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), sharepointDefaultBatchSize)
+ if batchSize <= 0 {
+ batchSize = sharepointDefaultBatchSize
+ }
+
+ return &SharePointConnector{
+ tenantID: strings.TrimSpace(stringConfig(credentials["tenant_id"])),
+ clientID: strings.TrimSpace(stringConfig(credentials["client_id"])),
+ clientSecret: stringConfig(credentials["client_secret"]),
+ siteURL: strings.TrimSpace(stringConfig(credentials["site_url"])),
+ batchSize: batchSize,
+ httpClient: &http.Client{Timeout: sharepointRequestTimeout},
+ tokenURL: "",
+ graphBaseURL: sharepointGraphBase,
+ }, nil
+}
+
+// Validate validates SharePoint connector settings, credentials, and access.
+func (c *SharePointConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return &ConnectorValidationError{Message: "SharePoint connector is nil"}
+ }
+ if c.tenantID == "" || c.clientID == "" || c.clientSecret == "" || c.siteURL == "" {
+ return &ConnectorMissingCredentialError{Message: "SharePoint credentials are incomplete: tenant_id, client_id, client_secret, and site_url are required"}
+ }
+ if c.batchSize <= 0 {
+ return &ConnectorValidationError{Message: "SharePoint connector batch_size must be a positive integer"}
+ }
+ if _, err := c.token(ctx); err != nil {
+ return &ConnectorMissingCredentialError{Message: fmt.Sprintf("Failed to acquire SharePoint access token: %v", err)}
+ }
+ if _, err := c.resolveSite(ctx); err != nil {
+ return c.siteValidationError(err)
+ }
+ return nil
+}
+
+// ValidateConnectorSetting validates SharePoint settings from an unsaved config.
+func (c *SharePointConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ return c.Validate(ctx)
+}
+
+// OpenSync opens one SharePoint sync session.
+func (c *SharePointConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ site, err := c.resolveSite(ctx)
+ if err != nil {
+ return nil, err
+ }
+ drives, err := c.listDrives(ctx, site.ID)
+ if err != nil {
+ return nil, err
+ }
+ // Drives are processed in ID order so checkpoints advance deterministically.
+ sort.SliceStable(drives, func(i, j int) bool {
+ return drives[i].ID < drives[j].ID
+ })
+ session := &sharePointSyncSession{
+ connector: c,
+ request: request,
+ site: site,
+ drives: drives,
+ batchSize: c.batchSize,
+ }
+ session.applyResume(request.Resume)
+ return session, nil
+}
+
+// OpenPrune opens one complete SharePoint prune snapshot session.
+func (c *SharePointConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ if err := c.Validate(ctx); err != nil {
+ return nil, err
+ }
+ site, err := c.resolveSite(ctx)
+ if err != nil {
+ return nil, err
+ }
+ drives, err := c.listDrives(ctx, site.ID)
+ if err != nil {
+ return nil, err
+ }
+ var documents []SlimDocument
+ for _, drive := range drives {
+ files, err := c.walkDriveFiles(ctx, drive.ID)
+ if err != nil {
+ return nil, err
+ }
+ for _, file := range files {
+ documents = append(documents, SlimDocument{SourceID: sharePointDocID(drive.ID, file.ID)})
+ }
+ }
+ return &sharePointPruneSession{documents: documents, batchSize: c.batchSize}, nil
+}
+
+// ---------------------------------------------------------------------------
+// Microsoft Graph client
+// ---------------------------------------------------------------------------
+
+type sharePointSite struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ WebURL string `json:"webUrl"`
+}
+
+type sharePointDrive struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ DriveType string `json:"driveType"`
+}
+
+type sharePointItem struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ Size int64 `json:"size"`
+ WebURL string `json:"webUrl"`
+ LastModifiedDateTime string `json:"lastModifiedDateTime"`
+ Folder *sharePointFolderFacet `json:"folder,omitempty"`
+ File *sharePointFileFacet `json:"file,omitempty"`
+}
+
+type sharePointFolderFacet struct {
+ ChildCount int `json:"childCount"`
+}
+
+type sharePointFileFacet struct {
+ MimeType string `json:"mimeType"`
+}
+
+type sharePointDrivesPage struct {
+ NextLink string `json:"@odata.nextLink"`
+ Value []sharePointDrive `json:"value"`
+}
+
+type sharePointChildrenPage struct {
+ NextLink string `json:"@odata.nextLink"`
+ Value []sharePointItem `json:"value"`
+}
+
+type sharePointTokenResponse struct {
+ AccessToken string `json:"access_token"`
+ ExpiresIn int `json:"expires_in"`
+}
+
+type sharePointCachedToken struct {
+ accessToken string
+ expiresAt time.Time
+}
+
+type sharePointGraphErrorBody struct {
+ Error struct {
+ Code string `json:"code"`
+ Message string `json:"message"`
+ } `json:"error"`
+}
+
+// sharePointAPIError marks a non-2xx Microsoft Graph response.
+type sharePointAPIError struct {
+ status int
+ body string
+}
+
+func (e *sharePointAPIError) Error() string {
+ return fmt.Sprintf("SharePoint Graph API request failed with status %d: %s", e.status, e.body)
+}
+
+// siteAPIPath converts a site URL such as
+// https://contoso.sharepoint.com/sites/MySite into the Graph relative path
+// /sites/contoso.sharepoint.com:/sites/MySite.
+func (c *SharePointConnector) siteAPIPath() string {
+ parsed, err := url.Parse(c.siteURL)
+ if err != nil || parsed.Host == "" {
+ return ""
+ }
+ pathPart := strings.Trim(parsed.Path, "/")
+ if pathPart == "" {
+ return "/sites/" + parsed.Host
+ }
+ segments := strings.Split(pathPart, "/")
+ escaped := make([]string, len(segments))
+ for i, segment := range segments {
+ escaped[i] = url.PathEscape(segment)
+ }
+ return "/sites/" + parsed.Host + ":/" + strings.Join(escaped, "/")
+}
+
+// resolveSite resolves the configured site URL through the Graph API.
+func (c *SharePointConnector) resolveSite(ctx context.Context) (*sharePointSite, error) {
+ apiPath := c.siteAPIPath()
+ if apiPath == "" {
+ return nil, &ConnectorValidationError{Message: fmt.Sprintf("Invalid SharePoint site URL %q", c.siteURL)}
+ }
+ var site sharePointSite
+ if err := c.getJSON(ctx, c.graphBaseURL+apiPath, &site); err != nil {
+ return nil, err
+ }
+ if site.ID == "" {
+ return nil, &ConnectorValidationError{Message: "Failed to access SharePoint site"}
+ }
+ return &site, nil
+}
+
+// listDrives lists every document library under the site, following
+// @odata.nextLink pagination.
+func (c *SharePointConnector) listDrives(ctx context.Context, siteID string) ([]sharePointDrive, error) {
+ var drives []sharePointDrive
+ apiURL := c.graphBaseURL + "/sites/" + url.PathEscape(siteID) + "/drives"
+ for apiURL != "" {
+ var page sharePointDrivesPage
+ if err := c.getJSON(ctx, apiURL, &page); err != nil {
+ return nil, err
+ }
+ drives = append(drives, page.Value...)
+ apiURL = page.NextLink
+ }
+ return drives, nil
+}
+
+// listChildren lists the children of a drive item; an empty itemID refers to
+// the drive root.
+func (c *SharePointConnector) listChildren(ctx context.Context, driveID, itemID string) ([]sharePointItem, error) {
+ var items []sharePointItem
+ var apiURL string
+ if itemID == "" {
+ apiURL = c.graphBaseURL + "/drives/" + url.PathEscape(driveID) + "/root/children"
+ } else {
+ apiURL = c.graphBaseURL + "/drives/" + url.PathEscape(driveID) + "/items/" + url.PathEscape(itemID) + "/children"
+ }
+ for apiURL != "" {
+ var page sharePointChildrenPage
+ if err := c.getJSON(ctx, apiURL, &page); err != nil {
+ return nil, err
+ }
+ items = append(items, page.Value...)
+ apiURL = page.NextLink
+ }
+ return items, nil
+}
+
+// walkDriveFiles depth-first walks a drive yielding file (non-folder) items.
+func (c *SharePointConnector) walkDriveFiles(ctx context.Context, driveID string) ([]sharePointItem, error) {
+ var files []sharePointItem
+ stack := []string{""}
+ for len(stack) > 0 {
+ folderID := stack[len(stack)-1]
+ stack = stack[:len(stack)-1]
+ children, err := c.listChildren(ctx, driveID, folderID)
+ if err != nil {
+ return nil, err
+ }
+ for _, child := range children {
+ if child.Folder != nil {
+ stack = append(stack, child.ID)
+ } else {
+ files = append(files, child)
+ }
+ }
+ }
+ return files, nil
+}
+
+// downloadContent downloads a file's raw bytes.
+func (c *SharePointConnector) downloadContent(ctx context.Context, driveID, itemID string) ([]byte, error) {
+ apiURL := c.graphBaseURL + "/drives/" + url.PathEscape(driveID) + "/items/" + url.PathEscape(itemID) + "/content"
+ return c.getBytes(ctx, apiURL)
+}
+
+// getJSON fetches and decodes one Graph JSON response.
+func (c *SharePointConnector) getJSON(ctx context.Context, apiURL string, out any) error {
+ body, err := c.doGraphRequest(ctx, apiURL, sharepointMaxJSONResponseSize)
+ if err != nil {
+ return err
+ }
+ return json.Unmarshal(body, out)
+}
+
+// getBytes fetches a raw Graph response body (file content).
+func (c *SharePointConnector) getBytes(ctx context.Context, apiURL string) ([]byte, error) {
+ return c.doGraphRequest(ctx, apiURL, sharepointMaxFileSize)
+}
+
+// doGraphRequest issues one authorized Graph GET, retrying transient failures
+// and refreshing an expired access token once on 401.
+func (c *SharePointConnector) doGraphRequest(ctx context.Context, apiURL string, maxBody int64) ([]byte, error) {
+ token, err := c.token(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ var lastErr error
+ retriedUnauthorized := false
+ for attempt := 1; attempt <= sharepointRetryCount; attempt++ {
+ requestCtx, cancel := context.WithTimeout(ctx, sharepointRequestTimeout)
+ req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, apiURL, nil)
+ if err != nil {
+ cancel()
+ return nil, err
+ }
+ req.Header.Set("Authorization", "Bearer "+token)
+
+ resp, err := c.httpClient.Do(req)
+ if err != nil {
+ cancel()
+ lastErr = err
+ } else {
+ body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxBody+1))
+ resp.Body.Close()
+ cancel()
+ if resp.StatusCode >= 400 {
+ lastErr = sharePointGraphError(resp.StatusCode, body)
+ if resp.StatusCode == http.StatusUnauthorized && !retriedUnauthorized {
+ c.invalidateToken(token)
+ token, err = c.token(ctx)
+ if err != nil {
+ return nil, err
+ }
+ retriedUnauthorized = true
+ attempt--
+ continue
+ }
+ if !sharePointRetryable(resp.StatusCode) {
+ return nil, lastErr
+ }
+ } else if readErr != nil {
+ return nil, readErr
+ } else if int64(len(body)) > maxBody {
+ return nil, fmt.Errorf("SharePoint API response from %s exceeds maximum size of %d bytes", apiURL, maxBody)
+ } else {
+ return body, nil
+ }
+ }
+ if attempt == sharepointRetryCount {
+ break
+ }
+ delay := time.Duration(attempt) * sharepointRetryBaseDelay
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-time.After(delay):
+ }
+ }
+ return nil, lastErr
+}
+
+func sharePointGraphError(status int, body []byte) error {
+ message := strings.TrimSpace(string(body))
+ var parsed sharePointGraphErrorBody
+ if json.Unmarshal(body, &parsed) == nil && parsed.Error.Message != "" {
+ message = parsed.Error.Message
+ }
+ return &sharePointAPIError{status: status, body: message}
+}
+
+func sharePointRetryable(status int) bool {
+ switch status {
+ case http.StatusTooManyRequests,
+ http.StatusInternalServerError,
+ http.StatusBadGateway,
+ http.StatusServiceUnavailable,
+ http.StatusGatewayTimeout:
+ return true
+ }
+ return false
+}
+
+// token returns a cached or freshly acquired app-only access token.
+func (c *SharePointConnector) token(ctx context.Context) (string, error) {
+ c.clientMu.Lock()
+ if c.accessToken != "" && !c.cachedTokenExpiredLocked() {
+ token := c.accessToken
+ c.clientMu.Unlock()
+ return token, nil
+ }
+ c.clientMu.Unlock()
+
+ cached, err := c.requestAccessToken(ctx)
+ if err != nil {
+ return "", err
+ }
+ if cached.accessToken == "" {
+ return "", &ConnectorMissingCredentialError{Message: "SharePoint token endpoint returned an empty access token"}
+ }
+
+ c.clientMu.Lock()
+ c.accessToken = cached.accessToken
+ c.tokenExpiry = cached.expiresAt
+ c.clientMu.Unlock()
+ return cached.accessToken, nil
+}
+
+func (c *SharePointConnector) cachedTokenExpiredLocked() bool {
+ return c.tokenExpiry.IsZero() || !c.currentTime().Add(sharepointTokenExpiryMargin).Before(c.tokenExpiry)
+}
+
+func (c *SharePointConnector) invalidateToken(token string) {
+ c.clientMu.Lock()
+ defer c.clientMu.Unlock()
+ if c.accessToken == token {
+ c.accessToken = ""
+ c.tokenExpiry = time.Time{}
+ }
+}
+
+func (c *SharePointConnector) currentTime() time.Time {
+ if c.now != nil {
+ return c.now()
+ }
+ return time.Now()
+}
+
+func (c *SharePointConnector) requestAccessToken(ctx context.Context) (sharePointCachedToken, error) {
+ tokenURL := c.tokenURL
+ if tokenURL == "" {
+ tokenURL = fmt.Sprintf(sharepointTokenURLFormat, url.PathEscape(c.tenantID))
+ }
+ form := url.Values{
+ "client_id": {c.clientID},
+ "client_secret": {c.clientSecret},
+ "grant_type": {"client_credentials"},
+ "scope": {sharepointGraphScope},
+ }
+ requestCtx, cancel := context.WithTimeout(ctx, sharepointRequestTimeout)
+ defer cancel()
+ req, err := http.NewRequestWithContext(requestCtx, http.MethodPost, tokenURL, strings.NewReader(form.Encode()))
+ if err != nil {
+ return sharePointCachedToken{}, err
+ }
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+ resp, err := c.httpClient.Do(req)
+ if err != nil {
+ return sharePointCachedToken{}, err
+ }
+ defer resp.Body.Close()
+ const maxTokenResponseSize = 1 << 20
+ body, err := io.ReadAll(io.LimitReader(resp.Body, maxTokenResponseSize+1))
+ if err != nil {
+ return sharePointCachedToken{}, err
+ }
+ if len(body) > maxTokenResponseSize {
+ return sharePointCachedToken{}, fmt.Errorf("SharePoint token response exceeds maximum size of %d bytes", maxTokenResponseSize)
+ }
+ if resp.StatusCode >= 400 {
+ message := strings.TrimSpace(string(body))
+ var parsed sharePointGraphErrorBody
+ if json.Unmarshal(body, &parsed) == nil && parsed.Error.Message != "" {
+ message = parsed.Error.Message
+ }
+ return sharePointCachedToken{}, &ConnectorMissingCredentialError{Message: fmt.Sprintf("Failed to acquire SharePoint access token: %s", message)}
+ }
+ var token sharePointTokenResponse
+ if err := json.Unmarshal(body, &token); err != nil {
+ return sharePointCachedToken{}, err
+ }
+ if token.ExpiresIn <= 0 {
+ return sharePointCachedToken{}, fmt.Errorf("SharePoint token endpoint returned invalid expires_in")
+ }
+ return sharePointCachedToken{
+ accessToken: token.AccessToken,
+ expiresAt: c.currentTime().Add(time.Duration(token.ExpiresIn) * time.Second),
+ }, nil
+}
+
+// siteValidationError maps Graph failures during settings validation.
+func (c *SharePointConnector) siteValidationError(err error) error {
+ var apiErr *sharePointAPIError
+ if errors.As(err, &apiErr) {
+ if apiErr.status == http.StatusUnauthorized || apiErr.status == http.StatusForbidden {
+ return &ConnectorValidationError{Message: "Invalid credentials or insufficient permissions for SharePoint"}
+ }
+ }
+ return &ConnectorValidationError{Message: fmt.Sprintf("SharePoint validation error: %v", err)}
+}
+
+// ---------------------------------------------------------------------------
+// Document building
+// ---------------------------------------------------------------------------
+
+// sharePointDocID namespaces a Graph driveItem ID by its drive ID because
+// driveItem IDs are only unique within a single drive.
+func sharePointDocID(driveID, itemID string) string {
+ return driveID + ":" + itemID
+}
+
+// sharePointExtension derives the file extension (with leading dot) from a
+// file name, mirroring the connector's document shape.
+func sharePointExtension(name string) string {
+ if index := strings.LastIndex(name, "."); index >= 0 && index < len(name)-1 {
+ return name[index:]
+ }
+ return ""
+}
+
+func (s *sharePointSyncSession) fileToDocument(ctx context.Context, drive sharePointDrive, file sharePointItem) (*SourceDocument, error) {
+ name := file.Name
+ if name == "" {
+ name = file.ID
+ }
+ blob, err := s.connector.downloadContent(ctx, drive.ID, file.ID)
+ if err != nil {
+ return nil, err
+ }
+
+ sizeBytes := file.Size
+ if sizeBytes <= 0 {
+ sizeBytes = int64(len(blob))
+ }
+
+ modified := parseOutlookTime(file.LastModifiedDateTime)
+ if modified.IsZero() {
+ modified = time.Now().UTC()
+ }
+
+ metadata := map[string]any{
+ "drive": drive.Name,
+ "drive_id": drive.ID,
+ "drive_item_id": file.ID,
+ }
+ if file.WebURL != "" {
+ metadata["web_url"] = file.WebURL
+ }
+
+ return &SourceDocument{
+ SourceID: sharePointDocID(drive.ID, file.ID),
+ SemanticIdentifier: name,
+ Extension: sharePointExtension(name),
+ Blob: blob,
+ UpdatedAt: modified,
+ SizeBytes: sizeBytes,
+ Metadata: metadata,
+ }, nil
+}
+
+// ---------------------------------------------------------------------------
+// Sync session with checkpoint resume
+// ---------------------------------------------------------------------------
+
+type sharePointSyncSession struct {
+ connector *SharePointConnector
+ request SyncRequest
+ site *sharePointSite
+ drives []sharePointDrive
+ batchSize int
+
+ resumeAfterDriveID string
+ driveIndex int
+ pending []SourceDocument
+ pendingCheckpoint *SyncCheckpoint
+ done bool
+}
+
+// NextBatch returns the next SharePoint document batch.
+func (s *sharePointSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ for {
+ if s.done {
+ return SyncBatch{}, io.EOF
+ }
+ if len(s.pending) == 0 {
+ if s.driveIndex >= len(s.drives) {
+ s.done = true
+ return SyncBatch{}, io.EOF
+ }
+ drive := s.drives[s.driveIndex]
+ s.driveIndex++
+ if drive.ID <= s.resumeAfterDriveID {
+ continue
+ }
+ documents, err := s.driveDocuments(ctx, drive)
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ if len(documents) == 0 {
+ continue
+ }
+ s.pending = documents
+ updatedAt := s.maxUpdatedAt(documents)
+ checkpoint := &SyncCheckpoint{
+ Cursor: fmt.Sprintf("sharepoint_drive_%s", drive.ID),
+ SourceID: fmt.Sprintf("sharepoint_drive_%s", drive.ID),
+ UpdatedAt: &updatedAt,
+ }
+ // The checkpoint only advances once the whole drive is flushed so
+ // an interrupted run re-processes a partially committed drive.
+ s.pendingCheckpoint = checkpoint
+ }
+ n := s.batchSize
+ if n > len(s.pending) {
+ n = len(s.pending)
+ }
+ chunk := s.pending[:n]
+ s.pending = s.pending[n:]
+ var checkpoint *SyncCheckpoint
+ if len(s.pending) == 0 && s.pendingCheckpoint != nil {
+ checkpoint = s.pendingCheckpoint
+ s.pendingCheckpoint = nil
+ }
+ return SyncBatch{Documents: chunk, Checkpoint: checkpoint}, nil
+ }
+}
+
+// Close closes the SharePoint sync session.
+func (s *sharePointSyncSession) Close() error {
+ return nil
+}
+
+func (s *sharePointSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+ if checkpoint == nil {
+ return
+ }
+ sourceID := firstNonEmpty(checkpoint.SourceID, checkpoint.Cursor)
+ const prefix = "sharepoint_drive_"
+ if !strings.HasPrefix(sourceID, prefix) {
+ return
+ }
+ driveID := strings.TrimPrefix(sourceID, prefix)
+ if driveID == "" {
+ return
+ }
+ s.resumeAfterDriveID = driveID
+}
+
+func (s *sharePointSyncSession) maxUpdatedAt(documents []SourceDocument) time.Time {
+ var max time.Time
+ for _, document := range documents {
+ if document.UpdatedAt.After(max) {
+ max = document.UpdatedAt
+ }
+ }
+ return max
+}
+
+// driveDocuments builds the documents for one drive, filtering files by the
+// sync window before downloading content.
+func (s *sharePointSyncSession) driveDocuments(ctx context.Context, drive sharePointDrive) ([]SourceDocument, error) {
+ files, err := s.connector.walkDriveFiles(ctx, drive.ID)
+ if err != nil {
+ return nil, err
+ }
+ var documents []SourceDocument
+ for _, file := range files {
+ if !s.fileInWindow(file) {
+ continue
+ }
+ doc, err := s.fileToDocument(ctx, drive, file)
+ if err != nil {
+ return nil, err
+ }
+ if doc != nil {
+ documents = append(documents, *doc)
+ }
+ }
+ return documents, nil
+}
+
+// fileInWindow reports whether a file's lastModifiedDateTime falls in the
+// exclusive-open, inclusive-closed sync window. Files without a parseable
+// modification time are always included.
+func (s *sharePointSyncSession) fileInWindow(file sharePointItem) bool {
+ modified := parseOutlookTime(file.LastModifiedDateTime)
+ if modified.IsZero() {
+ return true
+ }
+ if s.request.WindowStart != nil && !modified.After(*s.request.WindowStart) {
+ return false
+ }
+ if !s.request.WindowEnd.IsZero() && modified.After(s.request.WindowEnd) {
+ return false
+ }
+ return true
+}
+
+// ---------------------------------------------------------------------------
+// Prune session
+// ---------------------------------------------------------------------------
+
+type sharePointPruneSession struct {
+ documents []SlimDocument
+ batchSize int
+ offset int
+}
+
+// NextBatch returns the next slim SharePoint snapshot batch.
+func (s *sharePointPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ if s.offset >= len(s.documents) {
+ return PruneBatch{}, io.EOF
+ }
+ n := s.batchSize
+ if n <= 0 || n > len(s.documents)-s.offset {
+ n = len(s.documents) - s.offset
+ }
+ batch := s.documents[s.offset : s.offset+n]
+ s.offset += n
+ return PruneBatch{Documents: batch}, nil
+}
+
+// Close closes the SharePoint prune session.
+func (s *sharePointPruneSession) Close() error {
+ return nil
+}
diff --git a/internal/syncer/connector/sharepoint_test.go b/internal/syncer/connector/sharepoint_test.go
new file mode 100644
index 0000000000..05ee017b90
--- /dev/null
+++ b/internal/syncer/connector/sharepoint_test.go
@@ -0,0 +1,682 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func mustSharePointConnector(t *testing.T, serverURL string, extra map[string]any) *SharePointConnector {
+ t.Helper()
+ config := map[string]any{
+ "batch_size": 2,
+ "credentials": map[string]any{
+ "tenant_id": "tenant",
+ "client_id": "client",
+ "client_secret": "secret",
+ "site_url": "https://contoso.sharepoint.com/sites/MySite",
+ },
+ }
+ for key, value := range extra {
+ config[key] = value
+ }
+ connector, err := NewSharePointConnector(config)
+ if err != nil {
+ t.Fatalf("NewSharePointConnector: %v", err)
+ }
+ if serverURL != "" {
+ connector.tokenURL = serverURL + "/oauth2/v2.0/token"
+ connector.graphBaseURL = serverURL + "/v1.0"
+ }
+ return connector
+}
+
+type sharePointTestFixtures struct {
+ tokenStatus int
+ tokenBody string
+
+ siteStatus int
+ siteBody string
+ siteCalls *atomic.Int64
+
+ drivesBody string
+ drivesBody2 string
+ drivesCalls *atomic.Int64
+
+ childrenByKey map[string]string
+ childrenPage2 map[string]string
+ childrenCalls *atomic.Int64
+
+ contentByKey map[string]string
+ contentCalls *atomic.Int64
+
+ tokenCalls *atomic.Int64
+}
+
+const sharePointDefaultSiteBody = `{"id":"site-1","name":"MySite","webUrl":"https://contoso.sharepoint.com/sites/MySite"}`
+
+func newTestSharePointServer(t *testing.T, fixtures *sharePointTestFixtures) *httptest.Server {
+ t.Helper()
+ if fixtures.tokenStatus == 0 {
+ fixtures.tokenStatus = http.StatusOK
+ }
+ if fixtures.tokenBody == "" {
+ fixtures.tokenBody = `{"access_token":"tok-1","expires_in":3600}`
+ }
+ if fixtures.siteStatus == 0 {
+ fixtures.siteStatus = http.StatusOK
+ }
+ if fixtures.siteBody == "" {
+ fixtures.siteBody = sharePointDefaultSiteBody
+ }
+ if fixtures.drivesBody == "" {
+ fixtures.drivesBody = `{"value":[]}`
+ }
+
+ mux := http.NewServeMux()
+ server := httptest.NewServer(mux)
+ t.Cleanup(server.Close)
+
+ mux.HandleFunc("/oauth2/v2.0/token", func(w http.ResponseWriter, r *http.Request) {
+ if fixtures.tokenCalls != nil {
+ fixtures.tokenCalls.Add(1)
+ }
+ w.WriteHeader(fixtures.tokenStatus)
+ w.Write([]byte(fixtures.tokenBody))
+ })
+
+ mux.HandleFunc("/v1.0/", func(w http.ResponseWriter, r *http.Request) {
+ rest := strings.TrimPrefix(r.URL.Path, "/v1.0/")
+ switch {
+ case strings.HasPrefix(rest, "sites/"):
+ if strings.HasSuffix(rest, "/drives") {
+ if fixtures.drivesCalls != nil {
+ fixtures.drivesCalls.Add(1)
+ }
+ if r.URL.Query().Get("page") == "2" && fixtures.drivesBody2 != "" {
+ w.Write([]byte(fixtures.drivesBody2))
+ return
+ }
+ w.Write([]byte(fixtures.drivesBody))
+ return
+ }
+ if fixtures.siteCalls != nil {
+ fixtures.siteCalls.Add(1)
+ }
+ w.WriteHeader(fixtures.siteStatus)
+ w.Write([]byte(fixtures.siteBody))
+ case strings.HasPrefix(rest, "drives/"):
+ parts := strings.Split(rest, "/")
+ if len(parts) < 4 {
+ http.NotFound(w, r)
+ return
+ }
+ driveID := parts[1]
+ switch parts[2] {
+ case "root":
+ if len(parts) != 4 || parts[3] != "children" {
+ http.NotFound(w, r)
+ return
+ }
+ if fixtures.childrenCalls != nil {
+ fixtures.childrenCalls.Add(1)
+ }
+ key := driveID + ":/"
+ if r.URL.Query().Get("page") == "2" && fixtures.childrenPage2[key] != "" {
+ w.Write([]byte(fixtures.childrenPage2[key]))
+ return
+ }
+ w.Write([]byte(fixtures.childrenByKey[key]))
+ case "items":
+ if len(parts) != 5 {
+ http.NotFound(w, r)
+ return
+ }
+ itemID := parts[3]
+ switch parts[4] {
+ case "children":
+ if fixtures.childrenCalls != nil {
+ fixtures.childrenCalls.Add(1)
+ }
+ key := driveID + ":" + itemID
+ if r.URL.Query().Get("page") == "2" && fixtures.childrenPage2[key] != "" {
+ w.Write([]byte(fixtures.childrenPage2[key]))
+ return
+ }
+ w.Write([]byte(fixtures.childrenByKey[key]))
+ case "content":
+ if fixtures.contentCalls != nil {
+ fixtures.contentCalls.Add(1)
+ }
+ w.Write([]byte(fixtures.contentByKey[driveID+":"+itemID]))
+ default:
+ http.NotFound(w, r)
+ }
+ default:
+ http.NotFound(w, r)
+ }
+ default:
+ http.NotFound(w, r)
+ }
+ })
+ return server
+}
+
+func drainSharePointSync(t *testing.T, session SyncSession) []SourceDocument {
+ t.Helper()
+ var documents []SourceDocument
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ return documents
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ documents = append(documents, batch.Documents...)
+ }
+}
+
+func sharePointFileJSON(itemID, name, webURL, modified string, size int64) string {
+ return `{"id":"` + itemID + `","name":"` + name + `","webUrl":"` + webURL +
+ `","lastModifiedDateTime":"` + modified + `","size":` + strconv.FormatInt(size, 10) +
+ `,"file":{"mimeType":"text/plain"}}`
+}
+
+func sharePointFolderJSON(itemID, name string) string {
+ return `{"id":"` + itemID + `","name":"` + name + `","folder":{"childCount":1}}`
+}
+
+func TestSharePointValidateMissingCredentials(t *testing.T) {
+ connector := mustSharePointConnector(t, "", map[string]any{
+ "credentials": map[string]any{"tenant_id": "t", "client_id": "c"},
+ })
+ err := connector.Validate(context.Background())
+ if err == nil {
+ t.Fatal("Validate succeeded, want error")
+ }
+ var missing *ConnectorMissingCredentialError
+ if !errors.As(err, &missing) {
+ t.Fatalf("error type = %T, want *ConnectorMissingCredentialError", err)
+ }
+}
+
+func TestSharePointValidateSuccess(t *testing.T) {
+ fixtures := &sharePointTestFixtures{
+ tokenCalls: &atomic.Int64{},
+ siteCalls: &atomic.Int64{},
+ }
+ server := newTestSharePointServer(t, fixtures)
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ if err := connector.Validate(context.Background()); err != nil {
+ t.Fatalf("Validate: %v", err)
+ }
+ if fixtures.tokenCalls.Load() != 1 {
+ t.Fatalf("token calls = %d, want 1", fixtures.tokenCalls.Load())
+ }
+ if fixtures.siteCalls.Load() != 1 {
+ t.Fatalf("site calls = %d, want 1", fixtures.siteCalls.Load())
+ }
+}
+
+func TestSharePointValidateTokenFailure(t *testing.T) {
+ fixtures := &sharePointTestFixtures{
+ tokenStatus: http.StatusUnauthorized,
+ tokenBody: `{"error":"invalid_client","error_description":"bad secret"}`,
+ }
+ server := newTestSharePointServer(t, fixtures)
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ err := connector.Validate(context.Background())
+ if err == nil {
+ t.Fatal("Validate succeeded, want error")
+ }
+ var missing *ConnectorMissingCredentialError
+ if !errors.As(err, &missing) {
+ t.Fatalf("error type = %T, want *ConnectorMissingCredentialError", err)
+ }
+ if !strings.Contains(err.Error(), "Failed to acquire SharePoint access token") {
+ t.Fatalf("error = %q, want token acquisition failure", err)
+ }
+}
+
+func TestSharePointValidateSiteForbidden(t *testing.T) {
+ fixtures := &sharePointTestFixtures{
+ siteStatus: http.StatusForbidden,
+ siteBody: `{"error":{"code":"accessDenied","message":"Access denied"}}`,
+ }
+ server := newTestSharePointServer(t, fixtures)
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ err := connector.Validate(context.Background())
+ if err == nil {
+ t.Fatal("Validate succeeded, want error")
+ }
+ var validation *ConnectorValidationError
+ if !errors.As(err, &validation) {
+ t.Fatalf("error type = %T, want *ConnectorValidationError", err)
+ }
+ if !strings.Contains(err.Error(), "Invalid credentials or insufficient permissions") {
+ t.Fatalf("error = %q, want permission message", err)
+ }
+}
+
+func TestSharePointValidateSiteNotFound(t *testing.T) {
+ fixtures := &sharePointTestFixtures{
+ siteStatus: http.StatusNotFound,
+ siteBody: `{"error":{"code":"itemNotFound","message":"Site not found"}}`,
+ }
+ server := newTestSharePointServer(t, fixtures)
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ err := connector.Validate(context.Background())
+ if err == nil {
+ t.Fatal("Validate succeeded, want error")
+ }
+ var validation *ConnectorValidationError
+ if !errors.As(err, &validation) {
+ t.Fatalf("error type = %T, want *ConnectorValidationError", err)
+ }
+ if !strings.Contains(err.Error(), "SharePoint validation error") {
+ t.Fatalf("error = %q, want validation error message", err)
+ }
+}
+
+func TestSharePointSiteAPIPath(t *testing.T) {
+ connector := mustSharePointConnector(t, "", map[string]any{
+ "credentials": map[string]any{
+ "tenant_id": "t", "client_id": "c", "client_secret": "s",
+ "site_url": "https://contoso.sharepoint.com/sites/MySite",
+ },
+ })
+ if got := connector.siteAPIPath(); got != "/sites/contoso.sharepoint.com:/sites/MySite" {
+ t.Fatalf("siteAPIPath() = %q", got)
+ }
+
+ connector.siteURL = "https://contoso.sharepoint.com/sites/MySite/"
+ if got := connector.siteAPIPath(); got != "/sites/contoso.sharepoint.com:/sites/MySite" {
+ t.Fatalf("siteAPIPath(trailing slash) = %q", got)
+ }
+
+ connector.siteURL = "https://contoso.sharepoint.com/"
+ if got := connector.siteAPIPath(); got != "/sites/contoso.sharepoint.com" {
+ t.Fatalf("siteAPIPath(root) = %q", got)
+ }
+
+ connector.siteURL = "not a url"
+ if got := connector.siteAPIPath(); got != "" {
+ t.Fatalf("siteAPIPath(invalid) = %q, want empty", got)
+ }
+}
+
+func TestSharePointOpenSyncWalksLibrariesAndDownloads(t *testing.T) {
+ fixtures := &sharePointTestFixtures{
+ drivesBody: `{"value":[
+ {"id":"drv-A","name":"Documents","driveType":"documentLibrary"},
+ {"id":"drv-B","name":"SiteAssets","driveType":"documentLibrary"}
+ ]}`,
+ childrenByKey: map[string]string{
+ "drv-A:/": `{"value":[
+ ` + sharePointFileJSON("f1", "readme.txt", "https://contoso.sharepoint.com/f1", "2026-01-01T12:00:00Z", 16) + `,
+ ` + sharePointFolderJSON("d2", "sub") + `
+ ]}`,
+ "drv-A:d2": `{"value":[` + sharePointFileJSON("f2", "report.md", "https://contoso.sharepoint.com/f2", "2026-02-01T12:00:00Z", 8) + `]}`,
+ "drv-B:/": `{"value":[` + sharePointFileJSON("f3", "logo.png", "https://contoso.sharepoint.com/f3", "2026-03-01T12:00:00Z", 5) + `]}`,
+ },
+ contentByKey: map[string]string{
+ "drv-A:f1": "hello sharepoint",
+ "drv-A:f2": "# Report",
+ "drv-B:f3": "PNGDATA",
+ },
+ contentCalls: &atomic.Int64{},
+ }
+ server := newTestSharePointServer(t, fixtures)
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ documents := drainSharePointSync(t, session)
+
+ if len(documents) != 3 {
+ t.Fatalf("documents = %d, want 3", len(documents))
+ }
+ byID := map[string]SourceDocument{}
+ for _, document := range documents {
+ byID[document.SourceID] = document
+ }
+ if len(byID) != 3 {
+ t.Fatalf("duplicate SourceIDs: %v", documents)
+ }
+ for _, wantID := range []string{"drv-A:f1", "drv-A:f2", "drv-B:f3"} {
+ if _, ok := byID[wantID]; !ok {
+ t.Fatalf("missing document %q in %v", wantID, byID)
+ }
+ }
+
+ first := byID["drv-A:f1"]
+ if string(first.Blob) != "hello sharepoint" {
+ t.Fatalf("f1 blob = %q", first.Blob)
+ }
+ if first.SemanticIdentifier != "readme.txt" {
+ t.Fatalf("f1 semantic = %q", first.SemanticIdentifier)
+ }
+ if first.Extension != ".txt" {
+ t.Fatalf("f1 extension = %q", first.Extension)
+ }
+ if first.SizeBytes != 16 {
+ t.Fatalf("f1 size = %d, want 16", first.SizeBytes)
+ }
+ if first.Metadata["drive"] != "Documents" || first.Metadata["drive_id"] != "drv-A" || first.Metadata["drive_item_id"] != "f1" {
+ t.Fatalf("f1 metadata = %v", first.Metadata)
+ }
+ if first.Metadata["web_url"] != "https://contoso.sharepoint.com/f1" {
+ t.Fatalf("f1 web_url = %v", first.Metadata["web_url"])
+ }
+ wantModified := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC)
+ if !first.UpdatedAt.Equal(wantModified) {
+ t.Fatalf("f1 updated_at = %v, want %v", first.UpdatedAt, wantModified)
+ }
+ if byID["drv-A:f2"].Extension != ".md" {
+ t.Fatalf("f2 extension = %q", byID["drv-A:f2"].Extension)
+ }
+ if fixtures.contentCalls.Load() != 3 {
+ t.Fatalf("content calls = %d, want 3", fixtures.contentCalls.Load())
+ }
+}
+
+func TestSharePointWindowFilter(t *testing.T) {
+ fixtures := &sharePointTestFixtures{
+ drivesBody: `{"value":[{"id":"drv-A","name":"Documents","driveType":"documentLibrary"}]}`,
+ childrenByKey: map[string]string{
+ "drv-A:/": `{"value":[
+ ` + sharePointFileJSON("f1", "jan.txt", "https://contoso.sharepoint.com/f1", "2026-01-01T12:00:00Z", 1) + `,
+ ` + sharePointFileJSON("f2", "feb.txt", "https://contoso.sharepoint.com/f2", "2026-02-01T12:00:00Z", 1) + `,
+ {"id":"f3","name":"no-time.txt","webUrl":"https://contoso.sharepoint.com/f3","size":1,"file":{"mimeType":"text/plain"}}
+ ]}`,
+ },
+ contentByKey: map[string]string{
+ "drv-A:f1": "a", "drv-A:f2": "b", "drv-A:f3": "c",
+ },
+ }
+ server := newTestSharePointServer(t, fixtures)
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ windowStart := time.Date(2026, 1, 15, 0, 0, 0, 0, time.UTC)
+ windowEnd := time.Date(2026, 3, 1, 0, 0, 0, 0, time.UTC)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ WindowStart: &windowStart,
+ WindowEnd: windowEnd,
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ documents := drainSharePointSync(t, session)
+ if len(documents) != 2 {
+ t.Fatalf("incremental documents = %d, want 2 (f2, f3)", len(documents))
+ }
+ ids := map[string]bool{}
+ for _, document := range documents {
+ ids[document.SourceID] = true
+ }
+ if !ids["drv-A:f2"] || !ids["drv-A:f3"] || ids["drv-A:f1"] {
+ t.Fatalf("incremental ids = %v", ids)
+ }
+
+ fullSession, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync(full): %v", err)
+ }
+ full := drainSharePointSync(t, fullSession)
+ if len(full) != 3 {
+ t.Fatalf("full documents = %d, want 3", len(full))
+ }
+}
+
+func TestSharePointResumeSkipsSyncedDrives(t *testing.T) {
+ fixtures := &sharePointTestFixtures{
+ drivesBody: `{"value":[
+ {"id":"drv-A","name":"A","driveType":"documentLibrary"},
+ {"id":"drv-B","name":"B","driveType":"documentLibrary"},
+ {"id":"drv-C","name":"C","driveType":"documentLibrary"}
+ ]}`,
+ childrenByKey: map[string]string{
+ "drv-A:/": `{"value":[` + sharePointFileJSON("a1", "a.txt", "https://c/a1", "2026-01-01T00:00:00Z", 1) + `]}`,
+ "drv-B:/": `{"value":[` + sharePointFileJSON("b1", "b.txt", "https://c/b1", "2026-01-01T00:00:00Z", 1) + `]}`,
+ "drv-C:/": `{"value":[` + sharePointFileJSON("c1", "c.txt", "https://c/c1", "2026-01-01T00:00:00Z", 1) + `]}`,
+ },
+ contentByKey: map[string]string{
+ "drv-A:a1": "a", "drv-B:b1": "b", "drv-C:c1": "c",
+ },
+ contentCalls: &atomic.Int64{},
+ }
+ server := newTestSharePointServer(t, fixtures)
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{
+ Resume: &SyncCheckpoint{Cursor: "sharepoint_drive_drv-A"},
+ })
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+
+ var checkpoints []*SyncCheckpoint
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ if batch.Checkpoint != nil {
+ checkpoints = append(checkpoints, batch.Checkpoint)
+ }
+ }
+ if len(checkpoints) != 2 {
+ t.Fatalf("checkpoints = %d, want 2", len(checkpoints))
+ }
+ if checkpoints[0].Cursor != "sharepoint_drive_drv-B" || checkpoints[1].Cursor != "sharepoint_drive_drv-C" {
+ t.Fatalf("checkpoint cursors = %q, %q", checkpoints[0].Cursor, checkpoints[1].Cursor)
+ }
+ if fixtures.contentCalls.Load() != 2 {
+ t.Fatalf("content calls = %d, want 2 (drv-A skipped)", fixtures.contentCalls.Load())
+ }
+}
+
+func TestSharePointCheckpointAdvancesAfterDriveFlush(t *testing.T) {
+ fixtures := &sharePointTestFixtures{
+ drivesBody: `{"value":[
+ {"id":"drv-A","name":"A","driveType":"documentLibrary"},
+ {"id":"drv-B","name":"B","driveType":"documentLibrary"}
+ ]}`,
+ childrenByKey: map[string]string{
+ "drv-A:/": `{"value":[` + sharePointFileJSON("a1", "a.txt", "https://c/a1", "2026-01-01T00:00:00Z", 1) + `]}`,
+ "drv-B:/": `{"value":[` + sharePointFileJSON("b1", "b.txt", "https://c/b1", "2026-01-01T00:00:00Z", 1) + `]}`,
+ },
+ contentByKey: map[string]string{"drv-A:a1": "a", "drv-B:b1": "b"},
+ }
+ server := newTestSharePointServer(t, fixtures)
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("first NextBatch: %v", err)
+ }
+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "drv-A:a1" {
+ t.Fatalf("first batch = %+v", first)
+ }
+ if first.Checkpoint == nil || first.Checkpoint.Cursor != "sharepoint_drive_drv-A" {
+ t.Fatalf("first checkpoint = %+v, want sharepoint_drive_drv-A", first.Checkpoint)
+ }
+
+ second, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("second NextBatch: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "drv-B:b1" {
+ t.Fatalf("second batch = %+v", second)
+ }
+ if second.Checkpoint == nil || second.Checkpoint.Cursor != "sharepoint_drive_drv-B" {
+ t.Fatalf("second checkpoint = %+v, want sharepoint_drive_drv-B", second.Checkpoint)
+ }
+
+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("final NextBatch err = %v, want io.EOF", err)
+ }
+}
+
+func TestSharePointOpenPruneListsIdsWithoutDownload(t *testing.T) {
+ fixtures := &sharePointTestFixtures{
+ drivesBody: `{"value":[
+ {"id":"drv-A","name":"Documents","driveType":"documentLibrary"},
+ {"id":"drv-B","name":"SiteAssets","driveType":"documentLibrary"}
+ ]}`,
+ childrenByKey: map[string]string{
+ "drv-A:/": `{"value":[
+ ` + sharePointFileJSON("f1", "a.txt", "https://c/f1", "2026-01-01T00:00:00Z", 1) + `,
+ ` + sharePointFolderJSON("d2", "sub") + `
+ ]}`,
+ "drv-A:d2": `{"value":[` + sharePointFileJSON("f2", "b.txt", "https://c/f2", "2026-01-01T00:00:00Z", 1) + `]}`,
+ "drv-B:/": `{"value":[` + sharePointFileJSON("f3", "c.txt", "https://c/f3", "2026-01-01T00:00:00Z", 1) + `]}`,
+ },
+ contentCalls: &atomic.Int64{},
+ }
+ server := newTestSharePointServer(t, fixtures)
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ prune, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune: %v", err)
+ }
+ var ids []string
+ for {
+ batch, err := prune.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch: %v", err)
+ }
+ for _, document := range batch.Documents {
+ ids = append(ids, document.SourceID)
+ }
+ }
+ if len(ids) != 3 {
+ t.Fatalf("slim ids = %v, want 3", ids)
+ }
+ seen := map[string]bool{}
+ for _, id := range ids {
+ seen[id] = true
+ }
+ for _, want := range []string{"drv-A:f1", "drv-A:f2", "drv-B:f3"} {
+ if !seen[want] {
+ t.Fatalf("missing slim id %q in %v", want, ids)
+ }
+ }
+ if fixtures.contentCalls.Load() != 0 {
+ t.Fatalf("content calls = %d, want 0", fixtures.contentCalls.Load())
+ }
+}
+
+func TestSharePointChildrenPagination(t *testing.T) {
+ fixtures := &sharePointTestFixtures{
+ drivesBody: `{"value":[{"id":"drv-A","name":"Documents","driveType":"documentLibrary"}]}`,
+ childrenPage2: map[string]string{
+ "drv-A:/": `{"value":[` + sharePointFileJSON("f2", "two.txt", "https://c/f2", "2026-01-01T00:00:00Z", 1) + `]}`,
+ },
+ contentByKey: map[string]string{"drv-A:f1": "one", "drv-A:f2": "two"},
+ }
+ server := newTestSharePointServer(t, fixtures)
+ fixtures.childrenByKey = map[string]string{
+ "drv-A:/": `{"@odata.nextLink":"` + server.URL + `/v1.0/drives/drv-A/root/children?page=2","value":[` +
+ sharePointFileJSON("f1", "one.txt", "https://c/f1", "2026-01-01T00:00:00Z", 1) + `]}`,
+ }
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ documents := drainSharePointSync(t, session)
+ if len(documents) != 2 {
+ t.Fatalf("documents = %d, want 2 (paginated)", len(documents))
+ }
+}
+
+func TestSharePointTokenCachedAcrossCalls(t *testing.T) {
+ fixtures := &sharePointTestFixtures{
+ tokenCalls: &atomic.Int64{},
+ drivesBody: `{"value":[{"id":"drv-A","name":"Documents","driveType":"documentLibrary"}]}`,
+ childrenByKey: map[string]string{
+ "drv-A:/": `{"value":[` + sharePointFileJSON("f1", "a.txt", "https://c/f1", "2026-01-01T00:00:00Z", 1) + `]}`,
+ },
+ contentByKey: map[string]string{"drv-A:f1": "a"},
+ }
+ server := newTestSharePointServer(t, fixtures)
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ drainSharePointSync(t, session)
+
+ if fixtures.tokenCalls.Load() != 1 {
+ t.Fatalf("token calls = %d, want 1 (cached)", fixtures.tokenCalls.Load())
+ }
+}
+
+func TestSharePointFileSizeCap(t *testing.T) {
+ origMax := sharepointMaxFileSize
+ sharepointMaxFileSize = 4
+ t.Cleanup(func() { sharepointMaxFileSize = origMax })
+
+ fixtures := &sharePointTestFixtures{
+ drivesBody: `{"value":[{"id":"drv-A","name":"Documents","driveType":"documentLibrary"}]}`,
+ childrenByKey: map[string]string{
+ "drv-A:/": `{"value":[` + sharePointFileJSON("f1", "big.txt", "https://c/f1", "2026-01-01T00:00:00Z", 100) + `]}`,
+ },
+ contentByKey: map[string]string{"drv-A:f1": "0123456789"},
+ }
+ server := newTestSharePointServer(t, fixtures)
+ connector := mustSharePointConnector(t, server.URL, nil)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{})
+ if err != nil {
+ t.Fatalf("OpenSync: %v", err)
+ }
+ if _, err := session.NextBatch(context.Background()); err == nil || !strings.Contains(err.Error(), "exceeds maximum size") {
+ t.Fatalf("NextBatch err = %v, want size limit error", err)
+ }
+}