Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(go-syncer): add OneDrive connector with lazy fetch and resume (#18968)

mkaaaddd8cb0b2367d
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index a095b56f4e..0a52456fcc 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -49,6 +49,7 @@ func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "imap", NewIMAPConnector)
registerBuiltIn(registry, "jira", NewJiraConnector)
registerBuiltIn(registry, "outlook", NewOutlookConnector)
+ registerBuiltIn(registry, "onedrive", NewOneDriveConnector)
registerBuiltIn(registry, "notion", NewNotionConnector)
registerBuiltIn(registry, "rest_api", NewRestAPIConnector)
registerBuiltIn(registry, "xquik", NewXquikConnector)
diff --git a/internal/syncer/connector/onedrive.go b/internal/syncer/connector/onedrive.go
new file mode 100644
index 0000000000..7b3c6b645d
--- /dev/null
+++ b/internal/syncer/connector/onedrive.go
@@ -0,0 +1,887 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+)
+
+const (
+ onedriveGraphBase = "https://graph.microsoft.com/v1.0"
+ onedriveTokenURLFormat = "https://login.microsoftonline.com/%s/oauth2/v2.0/token"
+ onedriveGraphScope = "https://graph.microsoft.com/.default"
+ onedriveDefaultBatchSize = 2
+ onedriveDefaultSizeThreshold = 20 * 1024 * 1024
+ onedriveRequestTimeout = 60 * time.Second
+ onedriveRetryCount = 4
+ onedriveRetryBaseDelay = 200 * time.Millisecond
+ onedriveTokenExpiryMargin = 5 * time.Minute
+ onedriveMaxJSONResponseSize = 16 * 1024 * 1024
+)
+
+// onedriveSupportedExtensions mirrors the extension set used by the Python
+// OneDrive connector.
+var onedriveSupportedExtensions = map[string]struct{}{
+ ".pdf": {},
+ ".docx": {},
+ ".doc": {},
+ ".xlsx": {},
+ ".xls": {},
+ ".pptx": {},
+ ".ppt": {},
+ ".txt": {},
+ ".md": {},
+ ".csv": {},
+}
+
+// OneDriveConnector reads files from OneDrive / OneDrive for Business through
+// the Microsoft Graph delta API with app-only client-credentials auth.
+type OneDriveConnector struct {
+ tenantID string
+ clientID string
+ clientSecret string
+ folderPath string
+ batchSize int
+ sizeThreshold int64
+ graphBaseURL string
+ tokenURL string
+
+ clientMu sync.Mutex
+ accessToken string
+ tokenExpiry time.Time
+ httpClient *http.Client
+ now func() time.Time
+
+ acquireAccessToken func(ctx context.Context) (string, error)
+ getJSON func(ctx context.Context, apiURL string, out any) error
+ getBytes func(ctx context.Context, apiURL string) ([]byte, error)
+}
+
+// NewOneDriveConnector creates a OneDrive connector from Python-compatible config.
+func NewOneDriveConnector(config map[string]any) (*OneDriveConnector, error) {
+ credentials := configAnyMap(config["credentials"])
+ folderPath, err := normalizeOneDriveFolderPath(stringConfig(config["folder_path"]))
+ if err != nil {
+ return nil, err
+ }
+ batchSize := oneDriveBatchSize(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])))
+ sizeThreshold := int64(configInt(config["size_threshold"], onedriveDefaultSizeThreshold))
+ if sizeThreshold <= 0 {
+ sizeThreshold = onedriveDefaultSizeThreshold
+ }
+ return &OneDriveConnector{
+ tenantID: strings.TrimSpace(stringConfig(credentials["tenant_id"])),
+ clientID: strings.TrimSpace(stringConfig(credentials["client_id"])),
+ clientSecret: stringConfig(credentials["client_secret"]),
+ folderPath: folderPath,
+ batchSize: batchSize,
+ sizeThreshold: sizeThreshold,
+ graphBaseURL: onedriveGraphBase,
+ tokenURL: "",
+ httpClient: &http.Client{Timeout: onedriveRequestTimeout},
+ now: time.Now,
+ }, nil
+}
+
+func oneDriveBatchSize(value string) int {
+ if strings.TrimSpace(value) == "" {
+ return onedriveDefaultBatchSize
+ }
+ parsed, err := strconv.Atoi(strings.TrimSpace(value))
+ if err != nil {
+ return onedriveDefaultBatchSize
+ }
+ return parsed
+}
+
+func normalizeOneDriveFolderPath(value string) (string, error) {
+ value = strings.TrimSpace(value)
+ if value == "" {
+ return "", nil
+ }
+ segments := make([]string, 0, strings.Count(value, "/")+1)
+ for _, segment := range strings.Split(value, "/") {
+ if segment == "" {
+ continue
+ }
+ if segment == ".." {
+ return "", &ConnectorValidationError{Message: "folder_path must not contain '..' segments."}
+ }
+ segments = append(segments, segment)
+ }
+ if len(segments) == 0 {
+ return "", nil
+ }
+ return "/" + strings.Join(segments, "/"), nil
+}
+
+// Validate validates OneDrive credentials, batch size, and Graph access.
+func (c *OneDriveConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return &ConnectorValidationError{Message: "OneDrive connector is nil"}
+ }
+ if c.tenantID == "" || c.clientID == "" || c.clientSecret == "" {
+ return &ConnectorMissingCredentialError{Message: "OneDrive credentials are incomplete: tenant_id, client_id, and client_secret are required"}
+ }
+ if c.batchSize <= 0 {
+ return &ConnectorValidationError{Message: "OneDrive connector batch_size must be a positive integer"}
+ }
+ if _, err := c.token(ctx); err != nil {
+ return &ConnectorMissingCredentialError{Message: fmt.Sprintf("Failed to acquire OneDrive access token: %v", err)}
+ }
+ var page struct {
+ Value []onedriveDrive `json:"value"`
+ }
+ if err := c.graphJSON(ctx, c.graphBaseURL+"/drives?$top=1", &page); err != nil {
+ var httpErr *onedriveHTTPError
+ if errors.As(err, &httpErr) {
+ switch httpErr.status {
+ case http.StatusUnauthorized:
+ return &ConnectorMissingCredentialError{Message: "OneDrive access token is invalid or expired."}
+ case http.StatusForbidden:
+ return &ConnectorValidationError{Message: "The service principal lacks the 'Files.Read.All' permission required by the OneDrive connector."}
+ default:
+ return &ConnectorValidationError{Message: fmt.Sprintf("OneDrive validation failed (HTTP %d): %s", httpErr.status, httpErr.body)}
+ }
+ }
+ return &ConnectorValidationError{Message: fmt.Sprintf("OneDrive validation error: %v", err)}
+ }
+ if page.Value == nil {
+ return &ConnectorValidationError{Message: "Unexpected response format from Microsoft Graph /drives."}
+ }
+ return nil
+}
+
+// ValidateConnectorSetting validates an unsaved connector config through a
+// freshly constructed connector so request-derived URLs and credentials are
+// used instead of state copied from the receiver.
+func (c *OneDriveConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ candidate, err := NewOneDriveConnector(request)
+ if err != nil {
+ return err
+ }
+ if c != nil {
+ candidate.httpClient = c.httpClient
+ candidate.now = c.now
+ candidate.graphBaseURL = c.graphBaseURL
+ candidate.tokenURL = c.tokenURL
+ }
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ return candidate.Validate(ctx)
+}
+
+// OpenSync opens one OneDrive sync session.
+func (c *OneDriveConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ drives, err := c.listDrives(ctx)
+ if err != nil {
+ return nil, err
+ }
+ session := &onedriveSyncSession{
+ connector: c,
+ request: request,
+ drives: drives,
+ batchSize: c.effectiveBatchSize(),
+ deltaLinks: map[string]string{},
+ }
+ if err := session.applyResume(request.Resume); err != nil {
+ return nil, err
+ }
+ return session, nil
+}
+
+// OpenPrune opens one complete OneDrive prune snapshot session.
+func (c *OneDriveConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ drives, err := c.listDrives(ctx)
+ if err != nil {
+ return nil, err
+ }
+ return &onedrivePruneSession{
+ connector: c,
+ drives: drives,
+ batchSize: c.effectiveBatchSize(),
+ }, nil
+}
+
+func (c *OneDriveConnector) effectiveBatchSize() int {
+ if c.batchSize > 0 {
+ return c.batchSize
+ }
+ return onedriveDefaultBatchSize
+}
+
+func (c *OneDriveConnector) listDrives(ctx context.Context) ([]string, error) {
+ var driveIDs []string
+ apiURL := c.graphBaseURL + "/drives"
+ for apiURL != "" {
+ var page onedriveDrivesPage
+ if err := c.graphJSON(ctx, apiURL, &page); err != nil {
+ return nil, err
+ }
+ for _, drive := range page.Value {
+ if strings.TrimSpace(drive.ID) != "" {
+ driveIDs = append(driveIDs, drive.ID)
+ }
+ }
+ apiURL = page.NextLink
+ }
+ return uniqueSorted(driveIDs), nil
+}
+
+func (c *OneDriveConnector) deltaPage(ctx context.Context, apiURL string) (onedriveDeltaPage, error) {
+ var page onedriveDeltaPage
+ err := c.graphJSON(ctx, apiURL, &page)
+ return page, err
+}
+
+func (c *OneDriveConnector) deltaURL(driveID string) string {
+ if c.folderPath == "" {
+ return fmt.Sprintf("%s/drives/%s/root/delta", c.graphBaseURL, url.PathEscape(driveID))
+ }
+ segments := strings.Split(strings.TrimPrefix(c.folderPath, "/"), "/")
+ escaped := make([]string, 0, len(segments))
+ for _, segment := range segments {
+ escaped = append(escaped, url.PathEscape(segment))
+ }
+ return fmt.Sprintf("%s/drives/%s/root:/%s:/delta", c.graphBaseURL, url.PathEscape(driveID), strings.Join(escaped, "/"))
+}
+
+// Fetch downloads a delayed OneDrive file body.
+func (c *OneDriveConnector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {
+ var fetch onedriveFetchReference
+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {
+ return nil, err
+ }
+ if fetch.DriveID == "" || fetch.ItemID == "" {
+ return nil, fmt.Errorf("onedrive fetch reference is incomplete")
+ }
+ if fetch.Size > c.sizeThreshold {
+ return nil, fmt.Errorf("%s exceeds size threshold of %d", firstNonEmpty(fetch.Name, fetch.ItemID), c.sizeThreshold)
+ }
+ return c.downloadContent(ctx, fetch.DriveID, fetch.ItemID)
+}
+
+func (c *OneDriveConnector) downloadContent(ctx context.Context, driveID, itemID string) ([]byte, error) {
+ apiURL := fmt.Sprintf("%s/drives/%s/items/%s/content", c.graphBaseURL, url.PathEscape(driveID), url.PathEscape(itemID))
+ return c.graphBytes(ctx, apiURL)
+}
+
+func (c *OneDriveConnector) isAcceptedItem(item onedriveDriveItem) bool {
+ if strings.TrimSpace(item.ID) == "" || item.File == nil || item.Deleted != nil || item.Removed != nil {
+ return false
+ }
+ extension := strings.ToLower(filepath.Ext(item.Name))
+ if _, ok := onedriveSupportedExtensions[extension]; !ok {
+ return false
+ }
+ if item.Size < 0 || (item.Size > 0 && item.Size > c.sizeThreshold) {
+ return false
+ }
+ return true
+}
+
+func (c *OneDriveConnector) sourceDocument(driveID string, item onedriveDriveItem) (SourceDocument, bool) {
+ if !c.isAcceptedItem(item) {
+ return SourceDocument{}, false
+ }
+ name := firstNonEmpty(item.Name, item.ID)
+ size := item.Size
+ if size < 0 {
+ size = 0
+ }
+ fetch, _ := json.Marshal(onedriveFetchReference{DriveID: driveID, ItemID: item.ID, Name: name, Size: size})
+ updatedAt := parseOutlookTime(item.LastModifiedDateTime)
+ if updatedAt.IsZero() {
+ updatedAt = c.currentTime().UTC()
+ }
+ return SourceDocument{
+ SourceID: item.ID,
+ SemanticIdentifier: name,
+ Extension: strings.ToLower(filepath.Ext(name)),
+ FetchRef: &FetchReference{Key: string(fetch), SizeHint: size},
+ UpdatedAt: updatedAt,
+ SizeBytes: size,
+ Metadata: map[string]any{
+ "drive_id": driveID,
+ "web_url": item.WebURL,
+ "created_by": item.CreatedBy.User.DisplayName,
+ },
+ Fingerprint: firstNonEmpty(item.ETag, item.CTag),
+ }, true
+}
+
+// graphJSON fetches and decodes one Graph JSON response.
+func (c *OneDriveConnector) graphJSON(ctx context.Context, apiURL string, out any) error {
+ if c.getJSON != nil {
+ return c.getJSON(ctx, apiURL, out)
+ }
+ body, err := c.doGraphRequest(ctx, apiURL, onedriveMaxJSONResponseSize)
+ if err != nil {
+ return err
+ }
+ return json.Unmarshal(body, out)
+}
+
+// graphBytes fetches a raw Graph response body.
+func (c *OneDriveConnector) graphBytes(ctx context.Context, apiURL string) ([]byte, error) {
+ if c.getBytes != nil {
+ return c.getBytes(ctx, apiURL)
+ }
+ return c.doGraphRequest(ctx, apiURL, c.sizeThreshold)
+}
+
+func (c *OneDriveConnector) doGraphRequest(ctx context.Context, apiURL string, maxBody int64) ([]byte, error) {
+ token, err := c.token(ctx)
+ if err != nil {
+ return nil, err
+ }
+ var lastErr error
+ retriedUnauthorized := false
+ for attempt := 1; attempt <= onedriveRetryCount; attempt++ {
+ requestCtx, cancel := context.WithTimeout(ctx, onedriveRequestTimeout)
+ req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, apiURL, nil)
+ if err != nil {
+ cancel()
+ return nil, err
+ }
+ req.Header.Set("Authorization", "Bearer "+token)
+ resp, err := c.httpClient.Do(req)
+ if err != nil {
+ cancel()
+ lastErr = err
+ } else {
+ body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxBody+1))
+ resp.Body.Close()
+ cancel()
+ if resp.StatusCode >= 400 {
+ lastErr = &onedriveHTTPError{status: resp.StatusCode, body: strings.TrimSpace(string(body))}
+ if resp.StatusCode == http.StatusUnauthorized && !retriedUnauthorized {
+ c.invalidateToken(token)
+ token, err = c.token(ctx)
+ if err != nil {
+ return nil, err
+ }
+ retriedUnauthorized = true
+ attempt--
+ continue
+ }
+ if !isOneDriveRetryable(resp.StatusCode) {
+ return nil, lastErr
+ }
+ } else {
+ if readErr != nil {
+ return nil, readErr
+ }
+ if int64(len(body)) > maxBody {
+ return nil, fmt.Errorf("OneDrive API response from %s exceeds maximum size of %d bytes", apiURL, maxBody)
+ }
+ return body, nil
+ }
+ }
+ if attempt == onedriveRetryCount {
+ break
+ }
+ delay := time.Duration(attempt) * onedriveRetryBaseDelay
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-time.After(delay):
+ }
+ }
+ return nil, lastErr
+}
+
+func (c *OneDriveConnector) token(ctx context.Context) (string, error) {
+ c.clientMu.Lock()
+ if c.accessToken != "" && !c.cachedTokenExpiredLocked() {
+ token := c.accessToken
+ c.clientMu.Unlock()
+ return token, nil
+ }
+ c.clientMu.Unlock()
+
+ var cached onedriveCachedToken
+ var err error
+ if c.acquireAccessToken != nil {
+ var token string
+ token, err = c.acquireAccessToken(ctx)
+ if err == nil {
+ cached = onedriveCachedToken{accessToken: token, expiresAt: c.currentTime().Add(time.Hour)}
+ }
+ } else {
+ cached, err = c.requestAccessToken(ctx)
+ }
+ if err != nil {
+ return "", err
+ }
+ if cached.accessToken == "" {
+ return "", fmt.Errorf("OneDrive token endpoint returned an empty access token")
+ }
+ c.clientMu.Lock()
+ c.accessToken = cached.accessToken
+ c.tokenExpiry = cached.expiresAt
+ c.clientMu.Unlock()
+ return cached.accessToken, nil
+}
+
+func (c *OneDriveConnector) cachedTokenExpiredLocked() bool {
+ return c.tokenExpiry.IsZero() || !c.currentTime().Add(onedriveTokenExpiryMargin).Before(c.tokenExpiry)
+}
+
+func (c *OneDriveConnector) invalidateToken(token string) {
+ c.clientMu.Lock()
+ defer c.clientMu.Unlock()
+ if c.accessToken == token {
+ c.accessToken = ""
+ c.tokenExpiry = time.Time{}
+ }
+}
+
+func (c *OneDriveConnector) currentTime() time.Time {
+ if c.now != nil {
+ return c.now()
+ }
+ return time.Now()
+}
+
+func (c *OneDriveConnector) requestAccessToken(ctx context.Context) (onedriveCachedToken, error) {
+ form := url.Values{
+ "client_id": {c.clientID},
+ "client_secret": {c.clientSecret},
+ "grant_type": {"client_credentials"},
+ "scope": {onedriveGraphScope},
+ }
+ requestCtx, cancel := context.WithTimeout(ctx, onedriveRequestTimeout)
+ defer cancel()
+ tokenURL := c.tokenURL
+ if tokenURL == "" {
+ tokenURL = fmt.Sprintf(onedriveTokenURLFormat, url.PathEscape(c.tenantID))
+ }
+ req, err := http.NewRequestWithContext(requestCtx, http.MethodPost, tokenURL, strings.NewReader(form.Encode()))
+ if err != nil {
+ return onedriveCachedToken{}, err
+ }
+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+ resp, err := c.httpClient.Do(req)
+ if err != nil {
+ return onedriveCachedToken{}, err
+ }
+ defer resp.Body.Close()
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ if resp.StatusCode >= 400 {
+ return onedriveCachedToken{}, &onedriveHTTPError{status: resp.StatusCode, body: strings.TrimSpace(string(body))}
+ }
+ var token onedriveTokenResponse
+ if err := json.Unmarshal(body, &token); err != nil {
+ return onedriveCachedToken{}, err
+ }
+ if token.ExpiresIn <= 0 {
+ return onedriveCachedToken{}, fmt.Errorf("OneDrive token endpoint returned invalid expires_in")
+ }
+ return onedriveCachedToken{
+ accessToken: token.AccessToken,
+ expiresAt: c.currentTime().Add(time.Duration(token.ExpiresIn) * time.Second),
+ }, nil
+}
+
+func isOneDriveRetryable(status int) bool {
+ return status == http.StatusTooManyRequests || status == http.StatusRequestTimeout || status >= 500
+}
+
+func includeOneDriveItem(request SyncRequest, item onedriveDriveItem) bool {
+ if request.FromBeginning {
+ return true
+ }
+ updatedAt := parseOutlookTime(item.LastModifiedDateTime)
+ if updatedAt.IsZero() {
+ return true
+ }
+ if len(request.Fingerprints) > 0 {
+ fingerprint := firstNonEmpty(item.ETag, item.CTag)
+ stored, ok := request.Fingerprints[item.ID]
+ return fingerprint == "" || !ok || stored == "" || stored != fingerprint
+ }
+ return !beforeOrAtWindowStart(updatedAt, request.WindowStart) && !afterWindowEnd(updatedAt, request.WindowEnd)
+}
+
+// onedriveSyncSession streams OneDrive delta pages and checkpoints after each
+// emitted document.
+type onedriveSyncSession struct {
+ connector *OneDriveConnector
+ request SyncRequest
+ drives []string
+ driveIndex int
+ pageURL string
+ batchSize int
+ deltaLinks map[string]string
+ buffer []onedriveBufferedDocument
+
+ resumePageURL string
+ resumeSourceID string
+ resumeOffset int
+ completedCurrent bool
+}
+
+// NextBatch returns the next OneDrive source document batch.
+func (s *onedriveSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ documents := make([]SourceDocument, 0, s.batchSize)
+ var checkpoint *SyncCheckpoint
+ if len(s.buffer) > 0 {
+ n := min(s.batchSize, len(s.buffer))
+ for _, buffered := range s.buffer[:n] {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
+ s.buffer = s.buffer[n:]
+ }
+ for len(documents) < s.batchSize {
+ if s.driveIndex >= len(s.drives) {
+ if len(documents) == 0 {
+ return SyncBatch{}, io.EOF
+ }
+ break
+ }
+ page, err := s.nextDocumentPage(ctx)
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ if len(page) == 0 && s.completedCurrent {
+ continue
+ }
+ remaining := s.batchSize - len(documents)
+ if len(page) > remaining {
+ for _, buffered := range page[:remaining] {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
+ s.buffer = append(s.buffer, page[remaining:]...)
+ break
+ }
+ for _, buffered := range page {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
+ }
+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil
+}
+
+// Close closes the OneDrive sync session.
+func (s *onedriveSyncSession) Close() error {
+ return nil
+}
+
+// Fetch downloads a delayed OneDrive file body for this sync session.
+func (s *onedriveSyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {
+ return s.connector.Fetch(ctx, ref)
+}
+
+func (s *onedriveSyncSession) nextDocumentPage(ctx context.Context) ([]onedriveBufferedDocument, error) {
+ s.completedCurrent = false
+ driveID := s.drives[s.driveIndex]
+ requestURL := s.pageURL
+ if requestURL == "" {
+ requestURL = s.connector.deltaURL(driveID)
+ }
+ page, err := s.connector.deltaPage(ctx, requestURL)
+ if err != nil {
+ return nil, err
+ }
+ if page.DeltaLink != "" {
+ s.deltaLinks[driveID] = page.DeltaLink
+ }
+
+ all := make([]onedriveBufferedDocument, 0, len(page.Value))
+ included := make([]onedriveBufferedDocument, 0, len(page.Value))
+ pageOffset := 0
+ for _, item := range page.Value {
+ if !s.connector.isAcceptedItem(item) {
+ continue
+ }
+ doc, ok := s.connector.sourceDocument(driveID, item)
+ if !ok {
+ continue
+ }
+ pageOffset++
+ buffered := onedriveBufferedDocument{
+ document: doc,
+ checkpoint: s.checkpoint(onedriveSyncCursor{DriveID: driveID, PageURL: requestURL, Offset: pageOffset, SourceID: doc.SourceID, DeltaLinks: s.deltaLinks}, doc),
+ offset: pageOffset,
+ }
+ all = append(all, buffered)
+ if includeOneDriveItem(s.request, item) {
+ included = append(included, buffered)
+ }
+ }
+ documents, err := s.filterResumedDocuments(requestURL, all, included)
+ if err != nil {
+ return nil, err
+ }
+ if page.NextLink != "" {
+ if page.NextLink == requestURL {
+ return nil, fmt.Errorf("onedrive sync pagination did not advance from %s", requestURL)
+ }
+ s.pageURL = page.NextLink
+ return documents, nil
+ }
+ s.advanceDrive()
+ s.completedCurrent = true
+ return documents, nil
+}
+
+func (s *onedriveSyncSession) applyResume(checkpoint *SyncCheckpoint) error {
+ if checkpoint == nil {
+ return nil
+ }
+ if checkpoint.Cursor == "" {
+ return fmt.Errorf("onedrive sync cursor is missing: %w", ErrSyncResumeInvalid)
+ }
+ var cursor onedriveSyncCursor
+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
+ return fmt.Errorf("onedrive sync cursor is invalid: %w", ErrSyncResumeInvalid)
+ }
+ sourceID := firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
+ if sourceID == "" || cursor.DriveID == "" || cursor.PageURL == "" {
+ return fmt.Errorf("onedrive sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)
+ }
+ if len(cursor.DeltaLinks) > 0 {
+ s.deltaLinks = cursor.DeltaLinks
+ }
+ for index, driveID := range s.drives {
+ if driveID != cursor.DriveID {
+ continue
+ }
+ s.driveIndex = index
+ s.pageURL = cursor.PageURL
+ s.resumePageURL = cursor.PageURL
+ s.resumeSourceID = sourceID
+ s.resumeOffset = cursor.Offset
+ return nil
+ }
+ return fmt.Errorf("onedrive resume drive %q was not found in the current listing: %w", cursor.DriveID, ErrSyncResumeInvalid)
+}
+
+func (s *onedriveSyncSession) filterResumedDocuments(pageURL string, all, included []onedriveBufferedDocument) ([]onedriveBufferedDocument, error) {
+ if s.resumeSourceID == "" {
+ return included, nil
+ }
+ if pageURL != s.resumePageURL {
+ return nil, fmt.Errorf("onedrive resume page %q no longer matches checkpoint page %q: %w", pageURL, s.resumePageURL, ErrSyncResumeInvalid)
+ }
+ anchorOffset := -1
+ for _, candidate := range all {
+ if candidate.document.SourceID == s.resumeSourceID {
+ anchorOffset = candidate.offset
+ break
+ }
+ }
+ if anchorOffset < 0 {
+ return nil, fmt.Errorf("onedrive resume anchor %q was not found on %s: %w", s.resumeSourceID, pageURL, ErrSyncResumeInvalid)
+ }
+ s.clearResumeOffset()
+ filtered := included[:0]
+ for _, candidate := range included {
+ if candidate.offset > anchorOffset {
+ filtered = append(filtered, candidate)
+ }
+ }
+ return filtered, nil
+}
+
+func (s *onedriveSyncSession) checkpoint(cursor onedriveSyncCursor, doc SourceDocument) *SyncCheckpoint {
+ data, err := json.Marshal(cursor)
+ if err != nil {
+ return nil
+ }
+ updatedAt := doc.UpdatedAt
+ return &SyncCheckpoint{Cursor: string(data), UpdatedAt: &updatedAt, SourceID: doc.SourceID}
+}
+
+func (s *onedriveSyncSession) advanceDrive() {
+ s.driveIndex++
+ s.pageURL = ""
+ s.clearResumeOffset()
+}
+
+func (s *onedriveSyncSession) clearResumeOffset() {
+ s.resumePageURL = ""
+ s.resumeSourceID = ""
+ s.resumeOffset = 0
+}
+
+type onedriveBufferedDocument struct {
+ document SourceDocument
+ checkpoint *SyncCheckpoint
+ offset int
+}
+
+type onedriveSyncCursor struct {
+ DriveID string `json:"drive_id"`
+ PageURL string `json:"page_url"`
+ Offset int `json:"offset,omitempty"`
+ SourceID string `json:"source_id,omitempty"`
+ DeltaLinks map[string]string `json:"delta_links,omitempty"`
+}
+
+// onedrivePruneSession streams the complete OneDrive slim snapshot.
+type onedrivePruneSession struct {
+ connector *OneDriveConnector
+ drives []string
+ driveIndex int
+ pageURL string
+ batchSize int
+ buffer []SlimDocument
+}
+
+// NextBatch returns the next OneDrive prune snapshot batch.
+func (s *onedrivePruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ documents := make([]SlimDocument, 0, s.batchSize)
+ if len(s.buffer) > 0 {
+ n := min(s.batchSize, len(s.buffer))
+ documents = append(documents, s.buffer[:n]...)
+ s.buffer = s.buffer[n:]
+ }
+ for len(documents) < s.batchSize {
+ if s.driveIndex >= len(s.drives) {
+ if len(documents) == 0 {
+ return PruneBatch{}, io.EOF
+ }
+ break
+ }
+ page, err := s.nextSlimPage(ctx)
+ if err != nil {
+ return PruneBatch{}, err
+ }
+ remaining := s.batchSize - len(documents)
+ if len(page) > remaining {
+ documents = append(documents, page[:remaining]...)
+ s.buffer = append(s.buffer, page[remaining:]...)
+ break
+ }
+ documents = append(documents, page...)
+ }
+ return PruneBatch{Documents: documents}, nil
+}
+
+// Close closes the OneDrive prune session.
+func (s *onedrivePruneSession) Close() error {
+ return nil
+}
+
+func (s *onedrivePruneSession) nextSlimPage(ctx context.Context) ([]SlimDocument, error) {
+ driveID := s.drives[s.driveIndex]
+ requestURL := s.pageURL
+ if requestURL == "" {
+ requestURL = s.connector.deltaURL(driveID)
+ }
+ page, err := s.connector.deltaPage(ctx, requestURL)
+ if err != nil {
+ return nil, err
+ }
+ documents := make([]SlimDocument, 0, len(page.Value))
+ for _, item := range page.Value {
+ if s.connector.isAcceptedItem(item) {
+ documents = append(documents, SlimDocument{SourceID: item.ID})
+ }
+ }
+ if page.NextLink != "" {
+ if page.NextLink == requestURL {
+ return nil, fmt.Errorf("onedrive prune pagination did not advance from %s", requestURL)
+ }
+ s.pageURL = page.NextLink
+ } else {
+ s.driveIndex++
+ s.pageURL = ""
+ }
+ return documents, nil
+}
+
+type onedriveDrivesPage struct {
+ NextLink string `json:"@odata.nextLink"`
+ Value []onedriveDrive `json:"value"`
+}
+
+type onedriveDrive struct {
+ ID string `json:"id"`
+}
+
+type onedriveDeltaPage struct {
+ NextLink string `json:"@odata.nextLink"`
+ DeltaLink string `json:"@odata.deltaLink"`
+ Value []onedriveDriveItem `json:"value"`
+}
+
+type onedriveDriveItem struct {
+ ID string `json:"id"`
+ Name string `json:"name"`
+ Size int64 `json:"size"`
+ WebURL string `json:"webUrl"`
+ LastModifiedDateTime string `json:"lastModifiedDateTime"`
+ ETag string `json:"eTag"`
+ CTag string `json:"cTag"`
+ Folder *onedriveFacet `json:"folder,omitempty"`
+ File *onedriveFacet `json:"file,omitempty"`
+ Deleted map[string]any `json:"deleted,omitempty"`
+ Removed map[string]any `json:"@removed,omitempty"`
+ CreatedBy onedriveCreatedBy `json:"createdBy,omitempty"`
+}
+
+type onedriveFacet struct {
+ ChildCount int `json:"childCount"`
+}
+
+type onedriveCreatedBy struct {
+ User struct {
+ DisplayName string `json:"displayName"`
+ } `json:"user"`
+}
+
+type onedriveTokenResponse struct {
+ AccessToken string `json:"access_token"`
+ ExpiresIn int `json:"expires_in"`
+}
+
+type onedriveCachedToken struct {
+ accessToken string
+ expiresAt time.Time
+}
+
+type onedriveFetchReference struct {
+ DriveID string `json:"drive_id"`
+ ItemID string `json:"item_id"`
+ Name string `json:"name"`
+ Size int64 `json:"size"`
+}
+
+type onedriveHTTPError struct {
+ status int
+ body string
+}
+
+func (e *onedriveHTTPError) Error() string {
+ return fmt.Sprintf("OneDrive API returned HTTP %d: %s", e.status, e.body)
+}
diff --git a/internal/syncer/connector/onedrive_test.go b/internal/syncer/connector/onedrive_test.go
new file mode 100644
index 0000000000..1fe643f4de
--- /dev/null
+++ b/internal/syncer/connector/onedrive_test.go
@@ -0,0 +1,518 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestNewOneDriveConnectorParsesConfig(t *testing.T) {
+ connector, err := NewOneDriveConnector(map[string]any{
+ "credentials": map[string]any{
+ "tenant_id": "tenant",
+ "client_id": "client",
+ "client_secret": "secret",
+ },
+ "folder_path": " Documents/Reports/ ",
+ "batch_size": "0",
+ "sync_batch_size": "3",
+ "size_threshold": "123",
+ })
+ if err != nil {
+ t.Fatalf("NewOneDriveConnector failed: %v", err)
+ }
+ if connector.tenantID != "tenant" || connector.clientID != "client" || connector.clientSecret != "secret" {
+ t.Fatalf("credentials = %q/%q/%q", connector.tenantID, connector.clientID, connector.clientSecret)
+ }
+ if connector.folderPath != "/Documents/Reports" {
+ t.Fatalf("folder path = %q", connector.folderPath)
+ }
+ if connector.batchSize != 3 {
+ t.Fatalf("batch size = %d, want 3", connector.batchSize)
+ }
+ if connector.sizeThreshold != 123 {
+ t.Fatalf("size threshold = %d, want 123", connector.sizeThreshold)
+ }
+}
+
+func TestNewOneDriveConnectorDefaults(t *testing.T) {
+ connector, err := NewOneDriveConnector(nil)
+ if err != nil {
+ t.Fatalf("NewOneDriveConnector failed: %v", err)
+ }
+ if connector.batchSize != onedriveDefaultBatchSize {
+ t.Fatalf("batch size = %d, want %d", connector.batchSize, onedriveDefaultBatchSize)
+ }
+ if connector.sizeThreshold != onedriveDefaultSizeThreshold {
+ t.Fatalf("size threshold = %d, want %d", connector.sizeThreshold, onedriveDefaultSizeThreshold)
+ }
+ if connector.folderPath != "" {
+ t.Fatalf("folder path = %q, want empty", connector.folderPath)
+ }
+}
+
+func TestNewOneDriveConnectorRejectsParentFolderPath(t *testing.T) {
+ _, err := NewOneDriveConnector(map[string]any{"folder_path": "/Documents/../secret"})
+ if err == nil || !strings.Contains(err.Error(), "..") {
+ t.Fatalf("invalid folder path error = %v", err)
+ }
+}
+
+func TestNormalizeOneDriveFolderPath(t *testing.T) {
+ tests := []struct {
+ input string
+ want string
+ }{
+ {"", ""},
+ {"/", ""},
+ {"//", ""},
+ {"Documents/Reports", "/Documents/Reports"},
+ {"//Documents//Reports", "/Documents/Reports"},
+ {"/Documents/Reports/", "/Documents/Reports"},
+ }
+ for _, test := range tests {
+ got, err := normalizeOneDriveFolderPath(test.input)
+ if err != nil {
+ t.Fatalf("normalize %q: %v", test.input, err)
+ }
+ if got != test.want {
+ t.Fatalf("normalize %q = %q, want %q", test.input, got, test.want)
+ }
+ }
+}
+
+func TestOneDriveValidateRejectsMissingCredentials(t *testing.T) {
+ connector, _ := NewOneDriveConnector(nil)
+ err := connector.Validate(context.Background())
+ var missing *ConnectorMissingCredentialError
+ if !errors.As(err, &missing) {
+ t.Fatalf("Validate error = %v, want ConnectorMissingCredentialError", err)
+ }
+}
+
+func TestOneDriveValidateRejectsBatchSize(t *testing.T) {
+ connector := newOneDriveTestConnector()
+ connector.batchSize = 0
+ err := connector.Validate(context.Background())
+ var validation *ConnectorValidationError
+ if !errors.As(err, &validation) || !strings.Contains(err.Error(), "batch_size") {
+ t.Fatalf("Validate error = %v, want batch_size validation error", err)
+ }
+}
+
+func TestOneDriveValidateProbesDrives(t *testing.T) {
+ connector := newOneDriveTestConnector()
+ connector.acquireAccessToken = func(ctx context.Context) (string, error) { return "token", nil }
+ connector.getJSON = func(ctx context.Context, apiURL string, out any) error {
+ if !strings.Contains(apiURL, "/drives?$top=1") {
+ t.Fatalf("validation URL = %q, want /drives?$top=1", apiURL)
+ }
+ return json.Unmarshal([]byte(`{"value":[]}`), out)
+ }
+ if err := connector.Validate(context.Background()); err != nil {
+ t.Fatalf("Validate failed: %v", err)
+ }
+}
+
+func TestOneDriveValidateClassifiesGraphErrors(t *testing.T) {
+ tests := []struct {
+ status int
+ want any
+ }{
+ {http.StatusUnauthorized, &ConnectorMissingCredentialError{}},
+ {http.StatusForbidden, &ConnectorValidationError{}},
+ {http.StatusBadGateway, &ConnectorValidationError{}},
+ }
+ for _, test := range tests {
+ connector := newOneDriveTestConnector()
+ connector.acquireAccessToken = func(ctx context.Context) (string, error) { return "token", nil }
+ connector.getJSON = func(ctx context.Context, apiURL string, out any) error {
+ return &onedriveHTTPError{status: test.status, body: "graph error"}
+ }
+ err := connector.Validate(context.Background())
+ switch test.want.(type) {
+ case *ConnectorMissingCredentialError:
+ var missing *ConnectorMissingCredentialError
+ if !errors.As(err, &missing) {
+ t.Fatalf("status %d error = %v, want ConnectorMissingCredentialError", test.status, err)
+ }
+ case *ConnectorValidationError:
+ var validation *ConnectorValidationError
+ if !errors.As(err, &validation) {
+ t.Fatalf("status %d error = %v, want ConnectorValidationError", test.status, err)
+ }
+ }
+ }
+}
+
+func TestOneDriveValidateConnectorSettingUsesCandidateConfig(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch {
+ case r.Method == http.MethodPost && r.URL.Path == "/token":
+ if err := r.ParseForm(); err != nil {
+ t.Errorf("parse token form: %v", err)
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ if r.Form.Get("client_id") != "request-client" || r.Form.Get("client_secret") != "request-secret" {
+ t.Errorf("token credentials = %q/%q", r.Form.Get("client_id"), r.Form.Get("client_secret"))
+ w.WriteHeader(http.StatusUnauthorized)
+ return
+ }
+ w.Write([]byte(`{"access_token":"token","expires_in":3600}`))
+ case r.Method == http.MethodGet && r.URL.Path == "/v1.0/drives":
+ w.Write([]byte(`{"value":[]}`))
+ default:
+ t.Errorf("unexpected request %s %s", r.Method, r.URL.String())
+ w.WriteHeader(http.StatusNotFound)
+ }
+ }))
+ defer server.Close()
+
+ receiver := &OneDriveConnector{
+ httpClient: server.Client(),
+ now: time.Now,
+ graphBaseURL: server.URL + "/v1.0",
+ tokenURL: server.URL + "/token",
+ }
+ err := receiver.ValidateConnectorSetting(context.Background(), map[string]any{
+ "credentials": map[string]any{
+ "tenant_id": "request-tenant",
+ "client_id": "request-client",
+ "client_secret": "request-secret",
+ },
+ })
+ if err != nil {
+ t.Fatalf("ValidateConnectorSetting failed: %v", err)
+ }
+}
+
+func TestOneDriveOpenSyncUsesDeltaAndFetch(t *testing.T) {
+ connector := newOneDriveTestConnector()
+ connector.batchSize = 2
+ connector.getJSON = oneDriveDeltaStub(`{"value":[{"id":"drive-1"}]}`, `{"value":[
+ {"id":"file-1","name":"Report.PDF","file":{},"size":5,"webUrl":"https://example.test/report","lastModifiedDateTime":"2026-01-02T03:04:05Z","eTag":"etag-1","createdBy":{"user":{"displayName":"Alice"}}},
+ {"id":"photo","name":"photo.png","file":{},"size":5},
+ {"id":"folder","name":"Docs","folder":{}},
+ {"id":"deleted","name":"gone.pdf","file":{},"deleted":{"state":"deleted"}},
+ {"id":"file-2","name":"Notes.txt","file":{},"size":4,"lastModifiedDateTime":"2026-01-03T03:04:05Z","eTag":"etag-2"}
+ ],"@odata.deltaLink":"delta-1"}`)
+ var fetchedURL string
+ connector.getBytes = func(ctx context.Context, apiURL string) ([]byte, error) {
+ fetchedURL = apiURL
+ if !strings.Contains(apiURL, "/items/file-1/content") {
+ t.Fatalf("fetch URL = %q, want file-1 content", apiURL)
+ }
+ return []byte("pdf-body"), nil
+ }
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 2 {
+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))
+ }
+ doc := batch.Documents[0]
+ if doc.SourceID != "file-1" || doc.SemanticIdentifier != "Report.PDF" || doc.Extension != ".pdf" {
+ t.Fatalf("document shape = %+v", doc)
+ }
+ if !doc.UpdatedAt.Equal(mustTime(t, "2026-01-02T03:04:05Z")) {
+ t.Fatalf("updated at = %s", doc.UpdatedAt)
+ }
+ if doc.Fingerprint != "etag-1" {
+ t.Fatalf("fingerprint = %q, want etag-1", doc.Fingerprint)
+ }
+ if doc.Metadata["drive_id"] != "drive-1" || doc.Metadata["web_url"] != "https://example.test/report" || doc.Metadata["created_by"] != "Alice" {
+ t.Fatalf("metadata = %+v", doc.Metadata)
+ }
+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != "file-2" {
+ t.Fatalf("checkpoint = %+v, want file-2", batch.Checkpoint)
+ }
+
+ fetcher, ok := session.(Fetcher)
+ if !ok {
+ t.Fatalf("session does not implement Fetcher")
+ }
+ blob, err := fetcher.Fetch(context.Background(), *batch.Documents[0].FetchRef)
+ if err != nil {
+ t.Fatalf("Fetch failed: %v", err)
+ }
+ if string(blob) != "pdf-body" || fetchedURL == "" {
+ t.Fatalf("fetch blob = %q, url = %q", blob, fetchedURL)
+ }
+ if err := session.Close(); err != nil {
+ t.Fatalf("Close failed: %v", err)
+ }
+}
+
+func TestOneDriveOpenSyncWindowFilter(t *testing.T) {
+ connector := newOneDriveTestConnector()
+ connector.batchSize = 10
+ connector.getJSON = oneDriveDeltaStub(`{"value":[{"id":"drive-1"}]}`, `{"value":[
+ {"id":"before","name":"before.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-01T00:00:00Z","eTag":"e1"},
+ {"id":"inside","name":"inside.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-03T00:00:00Z","eTag":"e2"},
+ {"id":"after","name":"after.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-05T00:00:00Z","eTag":"e3"}
+ ]}`)
+ start := mustTime(t, "2026-01-02T00:00:00Z")
+ end := mustTime(t, "2026-01-04T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "inside" {
+ t.Fatalf("documents = %+v, want inside", batch.Documents)
+ }
+}
+
+func TestOneDriveOpenSyncFingerprintFilter(t *testing.T) {
+ connector := newOneDriveTestConnector()
+ connector.batchSize = 10
+ connector.getJSON = oneDriveDeltaStub(`{"value":[{"id":"drive-1"}]}`, `{"value":[
+ {"id":"changed","name":"changed.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-03T00:00:00Z","eTag":"new"},
+ {"id":"same","name":"same.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-03T00:00:00Z","eTag":"same"},
+ {"id":"missing","name":"missing.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-03T00:00:00Z","eTag":"missing"}
+ ]}`)
+ request := SyncRequest{
+ WindowStart: onedriveMustTimePointer(t, "2026-01-02T00:00:00Z"),
+ WindowEnd: mustTime(t, "2026-01-04T00:00:00Z"),
+ Fingerprints: map[string]string{
+ "changed": "old",
+ "same": "same",
+ },
+ }
+ session, err := connector.OpenSync(context.Background(), request)
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ got := []string{}
+ for _, doc := range batch.Documents {
+ got = append(got, doc.SourceID)
+ }
+ if len(got) != 2 || got[0] != "changed" || got[1] != "missing" {
+ t.Fatalf("documents = %v, want changed and missing", got)
+ }
+}
+
+func TestOneDriveOpenSyncResumeWithinPage(t *testing.T) {
+ connector := newOneDriveTestConnector()
+ connector.batchSize = 2
+ connector.getJSON = oneDriveDeltaStub(`{"value":[{"id":"drive-1"}]}`, `{"value":[
+ {"id":"file-1","name":"a.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-01T00:00:00Z","eTag":"e1"},
+ {"id":"file-2","name":"b.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-02T00:00:00Z","eTag":"e2"},
+ {"id":"file-3","name":"c.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-03T00:00:00Z","eTag":"e3"}
+ ],"@odata.deltaLink":"delta-1"}`)
+
+ first, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("first OpenSync failed: %v", err)
+ }
+ batch, err := first.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("first NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 2 || batch.Checkpoint == nil || batch.Checkpoint.SourceID != "file-2" {
+ t.Fatalf("first batch = %+v, want two docs ending at file-2", batch)
+ }
+
+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: batch.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync failed: %v", err)
+ }
+ second, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resume NextBatch failed: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "file-3" {
+ t.Fatalf("resume documents = %+v, want file-3", second.Documents)
+ }
+}
+
+func TestOneDriveOpenSyncResumeRejectsMissingCheckpoint(t *testing.T) {
+ connector := newOneDriveTestConnector()
+ connector.getJSON = oneDriveDeltaStub(`{"value":[{"id":"drive-1"}]}`, `{"value":[]}`)
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: &SyncCheckpoint{}})
+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)
+ }
+}
+
+func TestOneDriveOpenSyncResumeRejectsMissingAnchor(t *testing.T) {
+ connector := newOneDriveTestConnector()
+ connector.getJSON = oneDriveDeltaStub(`{"value":[{"id":"drive-1"}]}`, `{"value":[
+ {"id":"file-1","name":"a.pdf","file":{},"size":1,"eTag":"e1"},
+ {"id":"file-3","name":"c.pdf","file":{},"size":1,"eTag":"e3"}
+ ]}`)
+ checkpoint := &SyncCheckpoint{
+ Cursor: `{"drive_id":"drive-1","page_url":"` + connector.deltaURL("drive-1") + `","source_id":"file-2"}`,
+ }
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync failed: %v", err)
+ }
+ if _, err := session.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {
+ t.Fatalf("resume NextBatch err = %v, want ErrSyncResumeInvalid", err)
+ }
+}
+
+func TestOneDriveOpenPrune(t *testing.T) {
+ connector := newOneDriveTestConnector()
+ connector.batchSize = 1
+ firstURL := connector.deltaURL("drive-1")
+ secondURL := "https://graph.microsoft.com/v1.0/next"
+ connector.getJSON = func(ctx context.Context, apiURL string, out any) error {
+ switch apiURL {
+ case connector.graphBaseURL + "/drives":
+ return json.Unmarshal([]byte(`{"value":[{"id":"drive-1"}]}`), out)
+ case firstURL:
+ return json.Unmarshal([]byte(`{"value":[
+ {"id":"file-1","name":"a.pdf","file":{},"size":1},
+ {"id":"folder","name":"Docs","folder":{}},
+ {"id":"deleted","name":"gone.pdf","file":{},"deleted":{"state":"deleted"}},
+ {"id":"photo","name":"photo.png","file":{},"size":1}
+ ],"@odata.nextLink":"`+secondURL+`"}`), out)
+ case secondURL:
+ return json.Unmarshal([]byte(`{"value":[{"id":"file-2","name":"b.pdf","file":{},"size":1}]}`), out)
+ default:
+ t.Fatalf("unexpected prune URL %s", apiURL)
+ return nil
+ }
+ }
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune failed: %v", err)
+ }
+ var got []string
+ for {
+ batch, err := session.NextBatch(context.Background())
+ if errors.Is(err, io.EOF) {
+ break
+ }
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ for _, doc := range batch.Documents {
+ got = append(got, doc.SourceID)
+ }
+ }
+ if len(got) != 2 || got[0] != "file-1" || got[1] != "file-2" {
+ t.Fatalf("prune documents = %v, want file-1 and file-2", got)
+ }
+}
+
+func TestOneDrivePrunePaginationStall(t *testing.T) {
+ connector := newOneDriveTestConnector()
+ pageURL := connector.deltaURL("drive-1")
+ connector.getJSON = func(ctx context.Context, apiURL string, out any) error {
+ if apiURL == connector.graphBaseURL+"/drives" {
+ return json.Unmarshal([]byte(`{"value":[{"id":"drive-1"}]}`), out)
+ }
+ return json.Unmarshal([]byte(`{"value":[{"id":"file-1","name":"a.pdf","file":{},"size":1}],"@odata.nextLink":"`+pageURL+`"}`), out)
+ }
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune failed: %v", err)
+ }
+ if _, err := session.NextBatch(context.Background()); err == nil || !strings.Contains(err.Error(), "did not advance") {
+ t.Fatalf("prune NextBatch err = %v, want stalled pagination error", err)
+ }
+}
+
+func TestOneDriveFetch(t *testing.T) {
+ connector := newOneDriveTestConnector()
+ connector.sizeThreshold = 5
+ ref := FetchReference{Key: `{"drive_id":"drive-1","item_id":"item-1","name":"big.pdf","size":10}`}
+ if _, err := connector.Fetch(context.Background(), ref); err == nil || !strings.Contains(err.Error(), "exceeds size threshold") {
+ t.Fatalf("oversize Fetch err = %v, want size threshold error", err)
+ }
+
+ var fetchedURL string
+ connector.getBytes = func(ctx context.Context, apiURL string) ([]byte, error) {
+ fetchedURL = apiURL
+ if !strings.Contains(apiURL, "/items/item-1/content") {
+ t.Fatalf("fetch URL = %q, want item-1 content", apiURL)
+ }
+ return []byte("hello"), nil
+ }
+ ref.Key = `{"drive_id":"drive-1","item_id":"item-1","name":"ok.pdf","size":5}`
+ blob, err := connector.Fetch(context.Background(), ref)
+ if err != nil {
+ t.Fatalf("Fetch failed: %v", err)
+ }
+ if string(blob) != "hello" || fetchedURL == "" {
+ t.Fatalf("fetch blob = %q, url = %q", blob, fetchedURL)
+ }
+}
+
+func TestOneDriveDeltaURL(t *testing.T) {
+ connector := newOneDriveTestConnector()
+ if got := connector.deltaURL("drive 1"); got != "https://graph.microsoft.com/v1.0/drives/drive%201/root/delta" {
+ t.Fatalf("root delta URL = %q", got)
+ }
+ connector.folderPath = "/Documents/A B"
+ if got := connector.deltaURL("drive-1"); got != "https://graph.microsoft.com/v1.0/drives/drive-1/root:/Documents/A%20B:/delta" {
+ t.Fatalf("scoped delta URL = %q", got)
+ }
+}
+
+func newOneDriveTestConnector() *OneDriveConnector {
+ return &OneDriveConnector{
+ tenantID: "tenant",
+ clientID: "client",
+ clientSecret: "secret",
+ batchSize: onedriveDefaultBatchSize,
+ sizeThreshold: onedriveDefaultSizeThreshold,
+ graphBaseURL: onedriveGraphBase,
+ httpClient: http.DefaultClient,
+ now: time.Now,
+ }
+}
+
+func oneDriveDeltaStub(drivesJSON, deltaJSON string) func(ctx context.Context, apiURL string, out any) error {
+ return func(ctx context.Context, apiURL string, out any) error {
+ if strings.HasSuffix(apiURL, "/drives") {
+ return json.Unmarshal([]byte(drivesJSON), out)
+ }
+ return json.Unmarshal([]byte(deltaJSON), out)
+ }
+}
+
+func onedriveMustTimePointer(t *testing.T, value string) *time.Time {
+ t.Helper()
+ parsed := mustTime(t, value)
+ return &parsed
+}