RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(go-syncer): add OneDrive connector with lazy fetch and resume (#18968)
mkaaaddd8cb0b2367d
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex a095b56f4e..0a52456fcc 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -49,6 +49,7 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "imap", NewIMAPConnector)registerBuiltIn(registry, "jira", NewJiraConnector)registerBuiltIn(registry, "outlook", NewOutlookConnector)+ registerBuiltIn(registry, "onedrive", NewOneDriveConnector)registerBuiltIn(registry, "notion", NewNotionConnector)registerBuiltIn(registry, "rest_api", NewRestAPIConnector)registerBuiltIn(registry, "xquik", NewXquikConnector)diff --git a/internal/syncer/connector/onedrive.go b/internal/syncer/connector/onedrive.gonew file mode 100644index 0000000000..7b3c6b645d--- /dev/null+++ b/internal/syncer/connector/onedrive.go@@ -0,0 +1,887 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "path/filepath"+ "strconv"+ "strings"+ "sync"+ "time"+)++const (+ onedriveGraphBase = "https://graph.microsoft.com/v1.0"+ onedriveTokenURLFormat = "https://login.microsoftonline.com/%s/oauth2/v2.0/token"+ onedriveGraphScope = "https://graph.microsoft.com/.default"+ onedriveDefaultBatchSize = 2+ onedriveDefaultSizeThreshold = 20 * 1024 * 1024+ onedriveRequestTimeout = 60 * time.Second+ onedriveRetryCount = 4+ onedriveRetryBaseDelay = 200 * time.Millisecond+ onedriveTokenExpiryMargin = 5 * time.Minute+ onedriveMaxJSONResponseSize = 16 * 1024 * 1024+)++// onedriveSupportedExtensions mirrors the extension set used by the Python+// OneDrive connector.+var onedriveSupportedExtensions = map[string]struct{}{+ ".pdf": {},+ ".docx": {},+ ".doc": {},+ ".xlsx": {},+ ".xls": {},+ ".pptx": {},+ ".ppt": {},+ ".txt": {},+ ".md": {},+ ".csv": {},+}++// OneDriveConnector reads files from OneDrive / OneDrive for Business through+// the Microsoft Graph delta API with app-only client-credentials auth.+type OneDriveConnector struct {+ tenantID string+ clientID string+ clientSecret string+ folderPath string+ batchSize int+ sizeThreshold int64+ graphBaseURL string+ tokenURL string++ clientMu sync.Mutex+ accessToken string+ tokenExpiry time.Time+ httpClient *http.Client+ now func() time.Time++ acquireAccessToken func(ctx context.Context) (string, error)+ getJSON func(ctx context.Context, apiURL string, out any) error+ getBytes func(ctx context.Context, apiURL string) ([]byte, error)+}++// NewOneDriveConnector creates a OneDrive connector from Python-compatible config.+func NewOneDriveConnector(config map[string]any) (*OneDriveConnector, error) {+ credentials := configAnyMap(config["credentials"])+ folderPath, err := normalizeOneDriveFolderPath(stringConfig(config["folder_path"]))+ if err != nil {+ return nil, err+ }+ batchSize := oneDriveBatchSize(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])))+ sizeThreshold := int64(configInt(config["size_threshold"], onedriveDefaultSizeThreshold))+ if sizeThreshold <= 0 {+ sizeThreshold = onedriveDefaultSizeThreshold+ }+ return &OneDriveConnector{+ tenantID: strings.TrimSpace(stringConfig(credentials["tenant_id"])),+ clientID: strings.TrimSpace(stringConfig(credentials["client_id"])),+ clientSecret: stringConfig(credentials["client_secret"]),+ folderPath: folderPath,+ batchSize: batchSize,+ sizeThreshold: sizeThreshold,+ graphBaseURL: onedriveGraphBase,+ tokenURL: "",+ httpClient: &http.Client{Timeout: onedriveRequestTimeout},+ now: time.Now,+ }, nil+}++func oneDriveBatchSize(value string) int {+ if strings.TrimSpace(value) == "" {+ return onedriveDefaultBatchSize+ }+ parsed, err := strconv.Atoi(strings.TrimSpace(value))+ if err != nil {+ return onedriveDefaultBatchSize+ }+ return parsed+}++func normalizeOneDriveFolderPath(value string) (string, error) {+ value = strings.TrimSpace(value)+ if value == "" {+ return "", nil+ }+ segments := make([]string, 0, strings.Count(value, "/")+1)+ for _, segment := range strings.Split(value, "/") {+ if segment == "" {+ continue+ }+ if segment == ".." {+ return "", &ConnectorValidationError{Message: "folder_path must not contain '..' segments."}+ }+ segments = append(segments, segment)+ }+ if len(segments) == 0 {+ return "", nil+ }+ return "/" + strings.Join(segments, "/"), nil+}++// Validate validates OneDrive credentials, batch size, and Graph access.+func (c *OneDriveConnector) Validate(ctx context.Context) error {+ if c == nil {+ return &ConnectorValidationError{Message: "OneDrive connector is nil"}+ }+ if c.tenantID == "" || c.clientID == "" || c.clientSecret == "" {+ return &ConnectorMissingCredentialError{Message: "OneDrive credentials are incomplete: tenant_id, client_id, and client_secret are required"}+ }+ if c.batchSize <= 0 {+ return &ConnectorValidationError{Message: "OneDrive connector batch_size must be a positive integer"}+ }+ if _, err := c.token(ctx); err != nil {+ return &ConnectorMissingCredentialError{Message: fmt.Sprintf("Failed to acquire OneDrive access token: %v", err)}+ }+ var page struct {+ Value []onedriveDrive `json:"value"`+ }+ if err := c.graphJSON(ctx, c.graphBaseURL+"/drives?$top=1", &page); err != nil {+ var httpErr *onedriveHTTPError+ if errors.As(err, &httpErr) {+ switch httpErr.status {+ case http.StatusUnauthorized:+ return &ConnectorMissingCredentialError{Message: "OneDrive access token is invalid or expired."}+ case http.StatusForbidden:+ return &ConnectorValidationError{Message: "The service principal lacks the 'Files.Read.All' permission required by the OneDrive connector."}+ default:+ return &ConnectorValidationError{Message: fmt.Sprintf("OneDrive validation failed (HTTP %d): %s", httpErr.status, httpErr.body)}+ }+ }+ return &ConnectorValidationError{Message: fmt.Sprintf("OneDrive validation error: %v", err)}+ }+ if page.Value == nil {+ return &ConnectorValidationError{Message: "Unexpected response format from Microsoft Graph /drives."}+ }+ return nil+}++// ValidateConnectorSetting validates an unsaved connector config through a+// freshly constructed connector so request-derived URLs and credentials are+// used instead of state copied from the receiver.+func (c *OneDriveConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ candidate, err := NewOneDriveConnector(request)+ if err != nil {+ return err+ }+ if c != nil {+ candidate.httpClient = c.httpClient+ candidate.now = c.now+ candidate.graphBaseURL = c.graphBaseURL+ candidate.tokenURL = c.tokenURL+ }+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ return candidate.Validate(ctx)+}++// OpenSync opens one OneDrive sync session.+func (c *OneDriveConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ drives, err := c.listDrives(ctx)+ if err != nil {+ return nil, err+ }+ session := &onedriveSyncSession{+ connector: c,+ request: request,+ drives: drives,+ batchSize: c.effectiveBatchSize(),+ deltaLinks: map[string]string{},+ }+ if err := session.applyResume(request.Resume); err != nil {+ return nil, err+ }+ return session, nil+}++// OpenPrune opens one complete OneDrive prune snapshot session.+func (c *OneDriveConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ drives, err := c.listDrives(ctx)+ if err != nil {+ return nil, err+ }+ return &onedrivePruneSession{+ connector: c,+ drives: drives,+ batchSize: c.effectiveBatchSize(),+ }, nil+}++func (c *OneDriveConnector) effectiveBatchSize() int {+ if c.batchSize > 0 {+ return c.batchSize+ }+ return onedriveDefaultBatchSize+}++func (c *OneDriveConnector) listDrives(ctx context.Context) ([]string, error) {+ var driveIDs []string+ apiURL := c.graphBaseURL + "/drives"+ for apiURL != "" {+ var page onedriveDrivesPage+ if err := c.graphJSON(ctx, apiURL, &page); err != nil {+ return nil, err+ }+ for _, drive := range page.Value {+ if strings.TrimSpace(drive.ID) != "" {+ driveIDs = append(driveIDs, drive.ID)+ }+ }+ apiURL = page.NextLink+ }+ return uniqueSorted(driveIDs), nil+}++func (c *OneDriveConnector) deltaPage(ctx context.Context, apiURL string) (onedriveDeltaPage, error) {+ var page onedriveDeltaPage+ err := c.graphJSON(ctx, apiURL, &page)+ return page, err+}++func (c *OneDriveConnector) deltaURL(driveID string) string {+ if c.folderPath == "" {+ return fmt.Sprintf("%s/drives/%s/root/delta", c.graphBaseURL, url.PathEscape(driveID))+ }+ segments := strings.Split(strings.TrimPrefix(c.folderPath, "/"), "/")+ escaped := make([]string, 0, len(segments))+ for _, segment := range segments {+ escaped = append(escaped, url.PathEscape(segment))+ }+ return fmt.Sprintf("%s/drives/%s/root:/%s:/delta", c.graphBaseURL, url.PathEscape(driveID), strings.Join(escaped, "/"))+}++// Fetch downloads a delayed OneDrive file body.+func (c *OneDriveConnector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ var fetch onedriveFetchReference+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {+ return nil, err+ }+ if fetch.DriveID == "" || fetch.ItemID == "" {+ return nil, fmt.Errorf("onedrive fetch reference is incomplete")+ }+ if fetch.Size > c.sizeThreshold {+ return nil, fmt.Errorf("%s exceeds size threshold of %d", firstNonEmpty(fetch.Name, fetch.ItemID), c.sizeThreshold)+ }+ return c.downloadContent(ctx, fetch.DriveID, fetch.ItemID)+}++func (c *OneDriveConnector) downloadContent(ctx context.Context, driveID, itemID string) ([]byte, error) {+ apiURL := fmt.Sprintf("%s/drives/%s/items/%s/content", c.graphBaseURL, url.PathEscape(driveID), url.PathEscape(itemID))+ return c.graphBytes(ctx, apiURL)+}++func (c *OneDriveConnector) isAcceptedItem(item onedriveDriveItem) bool {+ if strings.TrimSpace(item.ID) == "" || item.File == nil || item.Deleted != nil || item.Removed != nil {+ return false+ }+ extension := strings.ToLower(filepath.Ext(item.Name))+ if _, ok := onedriveSupportedExtensions[extension]; !ok {+ return false+ }+ if item.Size < 0 || (item.Size > 0 && item.Size > c.sizeThreshold) {+ return false+ }+ return true+}++func (c *OneDriveConnector) sourceDocument(driveID string, item onedriveDriveItem) (SourceDocument, bool) {+ if !c.isAcceptedItem(item) {+ return SourceDocument{}, false+ }+ name := firstNonEmpty(item.Name, item.ID)+ size := item.Size+ if size < 0 {+ size = 0+ }+ fetch, _ := json.Marshal(onedriveFetchReference{DriveID: driveID, ItemID: item.ID, Name: name, Size: size})+ updatedAt := parseOutlookTime(item.LastModifiedDateTime)+ if updatedAt.IsZero() {+ updatedAt = c.currentTime().UTC()+ }+ return SourceDocument{+ SourceID: item.ID,+ SemanticIdentifier: name,+ Extension: strings.ToLower(filepath.Ext(name)),+ FetchRef: &FetchReference{Key: string(fetch), SizeHint: size},+ UpdatedAt: updatedAt,+ SizeBytes: size,+ Metadata: map[string]any{+ "drive_id": driveID,+ "web_url": item.WebURL,+ "created_by": item.CreatedBy.User.DisplayName,+ },+ Fingerprint: firstNonEmpty(item.ETag, item.CTag),+ }, true+}++// graphJSON fetches and decodes one Graph JSON response.+func (c *OneDriveConnector) graphJSON(ctx context.Context, apiURL string, out any) error {+ if c.getJSON != nil {+ return c.getJSON(ctx, apiURL, out)+ }+ body, err := c.doGraphRequest(ctx, apiURL, onedriveMaxJSONResponseSize)+ if err != nil {+ return err+ }+ return json.Unmarshal(body, out)+}++// graphBytes fetches a raw Graph response body.+func (c *OneDriveConnector) graphBytes(ctx context.Context, apiURL string) ([]byte, error) {+ if c.getBytes != nil {+ return c.getBytes(ctx, apiURL)+ }+ return c.doGraphRequest(ctx, apiURL, c.sizeThreshold)+}++func (c *OneDriveConnector) doGraphRequest(ctx context.Context, apiURL string, maxBody int64) ([]byte, error) {+ token, err := c.token(ctx)+ if err != nil {+ return nil, err+ }+ var lastErr error+ retriedUnauthorized := false+ for attempt := 1; attempt <= onedriveRetryCount; attempt++ {+ requestCtx, cancel := context.WithTimeout(ctx, onedriveRequestTimeout)+ req, err := http.NewRequestWithContext(requestCtx, http.MethodGet, apiURL, nil)+ if err != nil {+ cancel()+ return nil, err+ }+ req.Header.Set("Authorization", "Bearer "+token)+ resp, err := c.httpClient.Do(req)+ if err != nil {+ cancel()+ lastErr = err+ } else {+ body, readErr := io.ReadAll(io.LimitReader(resp.Body, maxBody+1))+ resp.Body.Close()+ cancel()+ if resp.StatusCode >= 400 {+ lastErr = &onedriveHTTPError{status: resp.StatusCode, body: strings.TrimSpace(string(body))}+ if resp.StatusCode == http.StatusUnauthorized && !retriedUnauthorized {+ c.invalidateToken(token)+ token, err = c.token(ctx)+ if err != nil {+ return nil, err+ }+ retriedUnauthorized = true+ attempt--+ continue+ }+ if !isOneDriveRetryable(resp.StatusCode) {+ return nil, lastErr+ }+ } else {+ if readErr != nil {+ return nil, readErr+ }+ if int64(len(body)) > maxBody {+ return nil, fmt.Errorf("OneDrive API response from %s exceeds maximum size of %d bytes", apiURL, maxBody)+ }+ return body, nil+ }+ }+ if attempt == onedriveRetryCount {+ break+ }+ delay := time.Duration(attempt) * onedriveRetryBaseDelay+ select {+ case <-ctx.Done():+ return nil, ctx.Err()+ case <-time.After(delay):+ }+ }+ return nil, lastErr+}++func (c *OneDriveConnector) token(ctx context.Context) (string, error) {+ c.clientMu.Lock()+ if c.accessToken != "" && !c.cachedTokenExpiredLocked() {+ token := c.accessToken+ c.clientMu.Unlock()+ return token, nil+ }+ c.clientMu.Unlock()++ var cached onedriveCachedToken+ var err error+ if c.acquireAccessToken != nil {+ var token string+ token, err = c.acquireAccessToken(ctx)+ if err == nil {+ cached = onedriveCachedToken{accessToken: token, expiresAt: c.currentTime().Add(time.Hour)}+ }+ } else {+ cached, err = c.requestAccessToken(ctx)+ }+ if err != nil {+ return "", err+ }+ if cached.accessToken == "" {+ return "", fmt.Errorf("OneDrive token endpoint returned an empty access token")+ }+ c.clientMu.Lock()+ c.accessToken = cached.accessToken+ c.tokenExpiry = cached.expiresAt+ c.clientMu.Unlock()+ return cached.accessToken, nil+}++func (c *OneDriveConnector) cachedTokenExpiredLocked() bool {+ return c.tokenExpiry.IsZero() || !c.currentTime().Add(onedriveTokenExpiryMargin).Before(c.tokenExpiry)+}++func (c *OneDriveConnector) invalidateToken(token string) {+ c.clientMu.Lock()+ defer c.clientMu.Unlock()+ if c.accessToken == token {+ c.accessToken = ""+ c.tokenExpiry = time.Time{}+ }+}++func (c *OneDriveConnector) currentTime() time.Time {+ if c.now != nil {+ return c.now()+ }+ return time.Now()+}++func (c *OneDriveConnector) requestAccessToken(ctx context.Context) (onedriveCachedToken, error) {+ form := url.Values{+ "client_id": {c.clientID},+ "client_secret": {c.clientSecret},+ "grant_type": {"client_credentials"},+ "scope": {onedriveGraphScope},+ }+ requestCtx, cancel := context.WithTimeout(ctx, onedriveRequestTimeout)+ defer cancel()+ tokenURL := c.tokenURL+ if tokenURL == "" {+ tokenURL = fmt.Sprintf(onedriveTokenURLFormat, url.PathEscape(c.tenantID))+ }+ req, err := http.NewRequestWithContext(requestCtx, http.MethodPost, tokenURL, strings.NewReader(form.Encode()))+ if err != nil {+ return onedriveCachedToken{}, err+ }+ req.Header.Set("Content-Type", "application/x-www-form-urlencoded")+ resp, err := c.httpClient.Do(req)+ if err != nil {+ return onedriveCachedToken{}, err+ }+ defer resp.Body.Close()+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ if resp.StatusCode >= 400 {+ return onedriveCachedToken{}, &onedriveHTTPError{status: resp.StatusCode, body: strings.TrimSpace(string(body))}+ }+ var token onedriveTokenResponse+ if err := json.Unmarshal(body, &token); err != nil {+ return onedriveCachedToken{}, err+ }+ if token.ExpiresIn <= 0 {+ return onedriveCachedToken{}, fmt.Errorf("OneDrive token endpoint returned invalid expires_in")+ }+ return onedriveCachedToken{+ accessToken: token.AccessToken,+ expiresAt: c.currentTime().Add(time.Duration(token.ExpiresIn) * time.Second),+ }, nil+}++func isOneDriveRetryable(status int) bool {+ return status == http.StatusTooManyRequests || status == http.StatusRequestTimeout || status >= 500+}++func includeOneDriveItem(request SyncRequest, item onedriveDriveItem) bool {+ if request.FromBeginning {+ return true+ }+ updatedAt := parseOutlookTime(item.LastModifiedDateTime)+ if updatedAt.IsZero() {+ return true+ }+ if len(request.Fingerprints) > 0 {+ fingerprint := firstNonEmpty(item.ETag, item.CTag)+ stored, ok := request.Fingerprints[item.ID]+ return fingerprint == "" || !ok || stored == "" || stored != fingerprint+ }+ return !beforeOrAtWindowStart(updatedAt, request.WindowStart) && !afterWindowEnd(updatedAt, request.WindowEnd)+}++// onedriveSyncSession streams OneDrive delta pages and checkpoints after each+// emitted document.+type onedriveSyncSession struct {+ connector *OneDriveConnector+ request SyncRequest+ drives []string+ driveIndex int+ pageURL string+ batchSize int+ deltaLinks map[string]string+ buffer []onedriveBufferedDocument++ resumePageURL string+ resumeSourceID string+ resumeOffset int+ completedCurrent bool+}++// NextBatch returns the next OneDrive source document batch.+func (s *onedriveSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ documents := make([]SourceDocument, 0, s.batchSize)+ var checkpoint *SyncCheckpoint+ if len(s.buffer) > 0 {+ n := min(s.batchSize, len(s.buffer))+ for _, buffered := range s.buffer[:n] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }+ s.buffer = s.buffer[n:]+ }+ for len(documents) < s.batchSize {+ if s.driveIndex >= len(s.drives) {+ if len(documents) == 0 {+ return SyncBatch{}, io.EOF+ }+ break+ }+ page, err := s.nextDocumentPage(ctx)+ if err != nil {+ return SyncBatch{}, err+ }+ if len(page) == 0 && s.completedCurrent {+ continue+ }+ remaining := s.batchSize - len(documents)+ if len(page) > remaining {+ for _, buffered := range page[:remaining] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }+ s.buffer = append(s.buffer, page[remaining:]...)+ break+ }+ for _, buffered := range page {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }+ }+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil+}++// Close closes the OneDrive sync session.+func (s *onedriveSyncSession) Close() error {+ return nil+}++// Fetch downloads a delayed OneDrive file body for this sync session.+func (s *onedriveSyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ return s.connector.Fetch(ctx, ref)+}++func (s *onedriveSyncSession) nextDocumentPage(ctx context.Context) ([]onedriveBufferedDocument, error) {+ s.completedCurrent = false+ driveID := s.drives[s.driveIndex]+ requestURL := s.pageURL+ if requestURL == "" {+ requestURL = s.connector.deltaURL(driveID)+ }+ page, err := s.connector.deltaPage(ctx, requestURL)+ if err != nil {+ return nil, err+ }+ if page.DeltaLink != "" {+ s.deltaLinks[driveID] = page.DeltaLink+ }++ all := make([]onedriveBufferedDocument, 0, len(page.Value))+ included := make([]onedriveBufferedDocument, 0, len(page.Value))+ pageOffset := 0+ for _, item := range page.Value {+ if !s.connector.isAcceptedItem(item) {+ continue+ }+ doc, ok := s.connector.sourceDocument(driveID, item)+ if !ok {+ continue+ }+ pageOffset+++ buffered := onedriveBufferedDocument{+ document: doc,+ checkpoint: s.checkpoint(onedriveSyncCursor{DriveID: driveID, PageURL: requestURL, Offset: pageOffset, SourceID: doc.SourceID, DeltaLinks: s.deltaLinks}, doc),+ offset: pageOffset,+ }+ all = append(all, buffered)+ if includeOneDriveItem(s.request, item) {+ included = append(included, buffered)+ }+ }+ documents, err := s.filterResumedDocuments(requestURL, all, included)+ if err != nil {+ return nil, err+ }+ if page.NextLink != "" {+ if page.NextLink == requestURL {+ return nil, fmt.Errorf("onedrive sync pagination did not advance from %s", requestURL)+ }+ s.pageURL = page.NextLink+ return documents, nil+ }+ s.advanceDrive()+ s.completedCurrent = true+ return documents, nil+}++func (s *onedriveSyncSession) applyResume(checkpoint *SyncCheckpoint) error {+ if checkpoint == nil {+ return nil+ }+ if checkpoint.Cursor == "" {+ return fmt.Errorf("onedrive sync cursor is missing: %w", ErrSyncResumeInvalid)+ }+ var cursor onedriveSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ return fmt.Errorf("onedrive sync cursor is invalid: %w", ErrSyncResumeInvalid)+ }+ sourceID := firstNonEmpty(cursor.SourceID, checkpoint.SourceID)+ if sourceID == "" || cursor.DriveID == "" || cursor.PageURL == "" {+ return fmt.Errorf("onedrive sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)+ }+ if len(cursor.DeltaLinks) > 0 {+ s.deltaLinks = cursor.DeltaLinks+ }+ for index, driveID := range s.drives {+ if driveID != cursor.DriveID {+ continue+ }+ s.driveIndex = index+ s.pageURL = cursor.PageURL+ s.resumePageURL = cursor.PageURL+ s.resumeSourceID = sourceID+ s.resumeOffset = cursor.Offset+ return nil+ }+ return fmt.Errorf("onedrive resume drive %q was not found in the current listing: %w", cursor.DriveID, ErrSyncResumeInvalid)+}++func (s *onedriveSyncSession) filterResumedDocuments(pageURL string, all, included []onedriveBufferedDocument) ([]onedriveBufferedDocument, error) {+ if s.resumeSourceID == "" {+ return included, nil+ }+ if pageURL != s.resumePageURL {+ return nil, fmt.Errorf("onedrive resume page %q no longer matches checkpoint page %q: %w", pageURL, s.resumePageURL, ErrSyncResumeInvalid)+ }+ anchorOffset := -1+ for _, candidate := range all {+ if candidate.document.SourceID == s.resumeSourceID {+ anchorOffset = candidate.offset+ break+ }+ }+ if anchorOffset < 0 {+ return nil, fmt.Errorf("onedrive resume anchor %q was not found on %s: %w", s.resumeSourceID, pageURL, ErrSyncResumeInvalid)+ }+ s.clearResumeOffset()+ filtered := included[:0]+ for _, candidate := range included {+ if candidate.offset > anchorOffset {+ filtered = append(filtered, candidate)+ }+ }+ return filtered, nil+}++func (s *onedriveSyncSession) checkpoint(cursor onedriveSyncCursor, doc SourceDocument) *SyncCheckpoint {+ data, err := json.Marshal(cursor)+ if err != nil {+ return nil+ }+ updatedAt := doc.UpdatedAt+ return &SyncCheckpoint{Cursor: string(data), UpdatedAt: &updatedAt, SourceID: doc.SourceID}+}++func (s *onedriveSyncSession) advanceDrive() {+ s.driveIndex+++ s.pageURL = ""+ s.clearResumeOffset()+}++func (s *onedriveSyncSession) clearResumeOffset() {+ s.resumePageURL = ""+ s.resumeSourceID = ""+ s.resumeOffset = 0+}++type onedriveBufferedDocument struct {+ document SourceDocument+ checkpoint *SyncCheckpoint+ offset int+}++type onedriveSyncCursor struct {+ DriveID string `json:"drive_id"`+ PageURL string `json:"page_url"`+ Offset int `json:"offset,omitempty"`+ SourceID string `json:"source_id,omitempty"`+ DeltaLinks map[string]string `json:"delta_links,omitempty"`+}++// onedrivePruneSession streams the complete OneDrive slim snapshot.+type onedrivePruneSession struct {+ connector *OneDriveConnector+ drives []string+ driveIndex int+ pageURL string+ batchSize int+ buffer []SlimDocument+}++// NextBatch returns the next OneDrive prune snapshot batch.+func (s *onedrivePruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ documents := make([]SlimDocument, 0, s.batchSize)+ if len(s.buffer) > 0 {+ n := min(s.batchSize, len(s.buffer))+ documents = append(documents, s.buffer[:n]...)+ s.buffer = s.buffer[n:]+ }+ for len(documents) < s.batchSize {+ if s.driveIndex >= len(s.drives) {+ if len(documents) == 0 {+ return PruneBatch{}, io.EOF+ }+ break+ }+ page, err := s.nextSlimPage(ctx)+ if err != nil {+ return PruneBatch{}, err+ }+ remaining := s.batchSize - len(documents)+ if len(page) > remaining {+ documents = append(documents, page[:remaining]...)+ s.buffer = append(s.buffer, page[remaining:]...)+ break+ }+ documents = append(documents, page...)+ }+ return PruneBatch{Documents: documents}, nil+}++// Close closes the OneDrive prune session.+func (s *onedrivePruneSession) Close() error {+ return nil+}++func (s *onedrivePruneSession) nextSlimPage(ctx context.Context) ([]SlimDocument, error) {+ driveID := s.drives[s.driveIndex]+ requestURL := s.pageURL+ if requestURL == "" {+ requestURL = s.connector.deltaURL(driveID)+ }+ page, err := s.connector.deltaPage(ctx, requestURL)+ if err != nil {+ return nil, err+ }+ documents := make([]SlimDocument, 0, len(page.Value))+ for _, item := range page.Value {+ if s.connector.isAcceptedItem(item) {+ documents = append(documents, SlimDocument{SourceID: item.ID})+ }+ }+ if page.NextLink != "" {+ if page.NextLink == requestURL {+ return nil, fmt.Errorf("onedrive prune pagination did not advance from %s", requestURL)+ }+ s.pageURL = page.NextLink+ } else {+ s.driveIndex+++ s.pageURL = ""+ }+ return documents, nil+}++type onedriveDrivesPage struct {+ NextLink string `json:"@odata.nextLink"`+ Value []onedriveDrive `json:"value"`+}++type onedriveDrive struct {+ ID string `json:"id"`+}++type onedriveDeltaPage struct {+ NextLink string `json:"@odata.nextLink"`+ DeltaLink string `json:"@odata.deltaLink"`+ Value []onedriveDriveItem `json:"value"`+}++type onedriveDriveItem struct {+ ID string `json:"id"`+ Name string `json:"name"`+ Size int64 `json:"size"`+ WebURL string `json:"webUrl"`+ LastModifiedDateTime string `json:"lastModifiedDateTime"`+ ETag string `json:"eTag"`+ CTag string `json:"cTag"`+ Folder *onedriveFacet `json:"folder,omitempty"`+ File *onedriveFacet `json:"file,omitempty"`+ Deleted map[string]any `json:"deleted,omitempty"`+ Removed map[string]any `json:"@removed,omitempty"`+ CreatedBy onedriveCreatedBy `json:"createdBy,omitempty"`+}++type onedriveFacet struct {+ ChildCount int `json:"childCount"`+}++type onedriveCreatedBy struct {+ User struct {+ DisplayName string `json:"displayName"`+ } `json:"user"`+}++type onedriveTokenResponse struct {+ AccessToken string `json:"access_token"`+ ExpiresIn int `json:"expires_in"`+}++type onedriveCachedToken struct {+ accessToken string+ expiresAt time.Time+}++type onedriveFetchReference struct {+ DriveID string `json:"drive_id"`+ ItemID string `json:"item_id"`+ Name string `json:"name"`+ Size int64 `json:"size"`+}++type onedriveHTTPError struct {+ status int+ body string+}++func (e *onedriveHTTPError) Error() string {+ return fmt.Sprintf("OneDrive API returned HTTP %d: %s", e.status, e.body)+}diff --git a/internal/syncer/connector/onedrive_test.go b/internal/syncer/connector/onedrive_test.gonew file mode 100644index 0000000000..1fe643f4de--- /dev/null+++ b/internal/syncer/connector/onedrive_test.go@@ -0,0 +1,518 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+ "time"+)++func TestNewOneDriveConnectorParsesConfig(t *testing.T) {+ connector, err := NewOneDriveConnector(map[string]any{+ "credentials": map[string]any{+ "tenant_id": "tenant",+ "client_id": "client",+ "client_secret": "secret",+ },+ "folder_path": " Documents/Reports/ ",+ "batch_size": "0",+ "sync_batch_size": "3",+ "size_threshold": "123",+ })+ if err != nil {+ t.Fatalf("NewOneDriveConnector failed: %v", err)+ }+ if connector.tenantID != "tenant" || connector.clientID != "client" || connector.clientSecret != "secret" {+ t.Fatalf("credentials = %q/%q/%q", connector.tenantID, connector.clientID, connector.clientSecret)+ }+ if connector.folderPath != "/Documents/Reports" {+ t.Fatalf("folder path = %q", connector.folderPath)+ }+ if connector.batchSize != 3 {+ t.Fatalf("batch size = %d, want 3", connector.batchSize)+ }+ if connector.sizeThreshold != 123 {+ t.Fatalf("size threshold = %d, want 123", connector.sizeThreshold)+ }+}++func TestNewOneDriveConnectorDefaults(t *testing.T) {+ connector, err := NewOneDriveConnector(nil)+ if err != nil {+ t.Fatalf("NewOneDriveConnector failed: %v", err)+ }+ if connector.batchSize != onedriveDefaultBatchSize {+ t.Fatalf("batch size = %d, want %d", connector.batchSize, onedriveDefaultBatchSize)+ }+ if connector.sizeThreshold != onedriveDefaultSizeThreshold {+ t.Fatalf("size threshold = %d, want %d", connector.sizeThreshold, onedriveDefaultSizeThreshold)+ }+ if connector.folderPath != "" {+ t.Fatalf("folder path = %q, want empty", connector.folderPath)+ }+}++func TestNewOneDriveConnectorRejectsParentFolderPath(t *testing.T) {+ _, err := NewOneDriveConnector(map[string]any{"folder_path": "/Documents/../secret"})+ if err == nil || !strings.Contains(err.Error(), "..") {+ t.Fatalf("invalid folder path error = %v", err)+ }+}++func TestNormalizeOneDriveFolderPath(t *testing.T) {+ tests := []struct {+ input string+ want string+ }{+ {"", ""},+ {"/", ""},+ {"//", ""},+ {"Documents/Reports", "/Documents/Reports"},+ {"//Documents//Reports", "/Documents/Reports"},+ {"/Documents/Reports/", "/Documents/Reports"},+ }+ for _, test := range tests {+ got, err := normalizeOneDriveFolderPath(test.input)+ if err != nil {+ t.Fatalf("normalize %q: %v", test.input, err)+ }+ if got != test.want {+ t.Fatalf("normalize %q = %q, want %q", test.input, got, test.want)+ }+ }+}++func TestOneDriveValidateRejectsMissingCredentials(t *testing.T) {+ connector, _ := NewOneDriveConnector(nil)+ err := connector.Validate(context.Background())+ var missing *ConnectorMissingCredentialError+ if !errors.As(err, &missing) {+ t.Fatalf("Validate error = %v, want ConnectorMissingCredentialError", err)+ }+}++func TestOneDriveValidateRejectsBatchSize(t *testing.T) {+ connector := newOneDriveTestConnector()+ connector.batchSize = 0+ err := connector.Validate(context.Background())+ var validation *ConnectorValidationError+ if !errors.As(err, &validation) || !strings.Contains(err.Error(), "batch_size") {+ t.Fatalf("Validate error = %v, want batch_size validation error", err)+ }+}++func TestOneDriveValidateProbesDrives(t *testing.T) {+ connector := newOneDriveTestConnector()+ connector.acquireAccessToken = func(ctx context.Context) (string, error) { return "token", nil }+ connector.getJSON = func(ctx context.Context, apiURL string, out any) error {+ if !strings.Contains(apiURL, "/drives?$top=1") {+ t.Fatalf("validation URL = %q, want /drives?$top=1", apiURL)+ }+ return json.Unmarshal([]byte(`{"value":[]}`), out)+ }+ if err := connector.Validate(context.Background()); err != nil {+ t.Fatalf("Validate failed: %v", err)+ }+}++func TestOneDriveValidateClassifiesGraphErrors(t *testing.T) {+ tests := []struct {+ status int+ want any+ }{+ {http.StatusUnauthorized, &ConnectorMissingCredentialError{}},+ {http.StatusForbidden, &ConnectorValidationError{}},+ {http.StatusBadGateway, &ConnectorValidationError{}},+ }+ for _, test := range tests {+ connector := newOneDriveTestConnector()+ connector.acquireAccessToken = func(ctx context.Context) (string, error) { return "token", nil }+ connector.getJSON = func(ctx context.Context, apiURL string, out any) error {+ return &onedriveHTTPError{status: test.status, body: "graph error"}+ }+ err := connector.Validate(context.Background())+ switch test.want.(type) {+ case *ConnectorMissingCredentialError:+ var missing *ConnectorMissingCredentialError+ if !errors.As(err, &missing) {+ t.Fatalf("status %d error = %v, want ConnectorMissingCredentialError", test.status, err)+ }+ case *ConnectorValidationError:+ var validation *ConnectorValidationError+ if !errors.As(err, &validation) {+ t.Fatalf("status %d error = %v, want ConnectorValidationError", test.status, err)+ }+ }+ }+}++func TestOneDriveValidateConnectorSettingUsesCandidateConfig(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch {+ case r.Method == http.MethodPost && r.URL.Path == "/token":+ if err := r.ParseForm(); err != nil {+ t.Errorf("parse token form: %v", err)+ w.WriteHeader(http.StatusBadRequest)+ return+ }+ if r.Form.Get("client_id") != "request-client" || r.Form.Get("client_secret") != "request-secret" {+ t.Errorf("token credentials = %q/%q", r.Form.Get("client_id"), r.Form.Get("client_secret"))+ w.WriteHeader(http.StatusUnauthorized)+ return+ }+ w.Write([]byte(`{"access_token":"token","expires_in":3600}`))+ case r.Method == http.MethodGet && r.URL.Path == "/v1.0/drives":+ w.Write([]byte(`{"value":[]}`))+ default:+ t.Errorf("unexpected request %s %s", r.Method, r.URL.String())+ w.WriteHeader(http.StatusNotFound)+ }+ }))+ defer server.Close()++ receiver := &OneDriveConnector{+ httpClient: server.Client(),+ now: time.Now,+ graphBaseURL: server.URL + "/v1.0",+ tokenURL: server.URL + "/token",+ }+ err := receiver.ValidateConnectorSetting(context.Background(), map[string]any{+ "credentials": map[string]any{+ "tenant_id": "request-tenant",+ "client_id": "request-client",+ "client_secret": "request-secret",+ },+ })+ if err != nil {+ t.Fatalf("ValidateConnectorSetting failed: %v", err)+ }+}++func TestOneDriveOpenSyncUsesDeltaAndFetch(t *testing.T) {+ connector := newOneDriveTestConnector()+ connector.batchSize = 2+ connector.getJSON = oneDriveDeltaStub(`{"value":[{"id":"drive-1"}]}`, `{"value":[+ {"id":"file-1","name":"Report.PDF","file":{},"size":5,"webUrl":"https://example.test/report","lastModifiedDateTime":"2026-01-02T03:04:05Z","eTag":"etag-1","createdBy":{"user":{"displayName":"Alice"}}},+ {"id":"photo","name":"photo.png","file":{},"size":5},+ {"id":"folder","name":"Docs","folder":{}},+ {"id":"deleted","name":"gone.pdf","file":{},"deleted":{"state":"deleted"}},+ {"id":"file-2","name":"Notes.txt","file":{},"size":4,"lastModifiedDateTime":"2026-01-03T03:04:05Z","eTag":"etag-2"}+ ],"@odata.deltaLink":"delta-1"}`)+ var fetchedURL string+ connector.getBytes = func(ctx context.Context, apiURL string) ([]byte, error) {+ fetchedURL = apiURL+ if !strings.Contains(apiURL, "/items/file-1/content") {+ t.Fatalf("fetch URL = %q, want file-1 content", apiURL)+ }+ return []byte("pdf-body"), nil+ }++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))+ }+ doc := batch.Documents[0]+ if doc.SourceID != "file-1" || doc.SemanticIdentifier != "Report.PDF" || doc.Extension != ".pdf" {+ t.Fatalf("document shape = %+v", doc)+ }+ if !doc.UpdatedAt.Equal(mustTime(t, "2026-01-02T03:04:05Z")) {+ t.Fatalf("updated at = %s", doc.UpdatedAt)+ }+ if doc.Fingerprint != "etag-1" {+ t.Fatalf("fingerprint = %q, want etag-1", doc.Fingerprint)+ }+ if doc.Metadata["drive_id"] != "drive-1" || doc.Metadata["web_url"] != "https://example.test/report" || doc.Metadata["created_by"] != "Alice" {+ t.Fatalf("metadata = %+v", doc.Metadata)+ }+ if batch.Checkpoint == nil || batch.Checkpoint.SourceID != "file-2" {+ t.Fatalf("checkpoint = %+v, want file-2", batch.Checkpoint)+ }++ fetcher, ok := session.(Fetcher)+ if !ok {+ t.Fatalf("session does not implement Fetcher")+ }+ blob, err := fetcher.Fetch(context.Background(), *batch.Documents[0].FetchRef)+ if err != nil {+ t.Fatalf("Fetch failed: %v", err)+ }+ if string(blob) != "pdf-body" || fetchedURL == "" {+ t.Fatalf("fetch blob = %q, url = %q", blob, fetchedURL)+ }+ if err := session.Close(); err != nil {+ t.Fatalf("Close failed: %v", err)+ }+}++func TestOneDriveOpenSyncWindowFilter(t *testing.T) {+ connector := newOneDriveTestConnector()+ connector.batchSize = 10+ connector.getJSON = oneDriveDeltaStub(`{"value":[{"id":"drive-1"}]}`, `{"value":[+ {"id":"before","name":"before.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-01T00:00:00Z","eTag":"e1"},+ {"id":"inside","name":"inside.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-03T00:00:00Z","eTag":"e2"},+ {"id":"after","name":"after.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-05T00:00:00Z","eTag":"e3"}+ ]}`)+ start := mustTime(t, "2026-01-02T00:00:00Z")+ end := mustTime(t, "2026-01-04T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "inside" {+ t.Fatalf("documents = %+v, want inside", batch.Documents)+ }+}++func TestOneDriveOpenSyncFingerprintFilter(t *testing.T) {+ connector := newOneDriveTestConnector()+ connector.batchSize = 10+ connector.getJSON = oneDriveDeltaStub(`{"value":[{"id":"drive-1"}]}`, `{"value":[+ {"id":"changed","name":"changed.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-03T00:00:00Z","eTag":"new"},+ {"id":"same","name":"same.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-03T00:00:00Z","eTag":"same"},+ {"id":"missing","name":"missing.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-03T00:00:00Z","eTag":"missing"}+ ]}`)+ request := SyncRequest{+ WindowStart: onedriveMustTimePointer(t, "2026-01-02T00:00:00Z"),+ WindowEnd: mustTime(t, "2026-01-04T00:00:00Z"),+ Fingerprints: map[string]string{+ "changed": "old",+ "same": "same",+ },+ }+ session, err := connector.OpenSync(context.Background(), request)+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ got := []string{}+ for _, doc := range batch.Documents {+ got = append(got, doc.SourceID)+ }+ if len(got) != 2 || got[0] != "changed" || got[1] != "missing" {+ t.Fatalf("documents = %v, want changed and missing", got)+ }+}++func TestOneDriveOpenSyncResumeWithinPage(t *testing.T) {+ connector := newOneDriveTestConnector()+ connector.batchSize = 2+ connector.getJSON = oneDriveDeltaStub(`{"value":[{"id":"drive-1"}]}`, `{"value":[+ {"id":"file-1","name":"a.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-01T00:00:00Z","eTag":"e1"},+ {"id":"file-2","name":"b.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-02T00:00:00Z","eTag":"e2"},+ {"id":"file-3","name":"c.pdf","file":{},"size":1,"lastModifiedDateTime":"2026-01-03T00:00:00Z","eTag":"e3"}+ ],"@odata.deltaLink":"delta-1"}`)++ first, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("first OpenSync failed: %v", err)+ }+ batch, err := first.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 || batch.Checkpoint == nil || batch.Checkpoint.SourceID != "file-2" {+ t.Fatalf("first batch = %+v, want two docs ending at file-2", batch)+ }++ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: batch.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "file-3" {+ t.Fatalf("resume documents = %+v, want file-3", second.Documents)+ }+}++func TestOneDriveOpenSyncResumeRejectsMissingCheckpoint(t *testing.T) {+ connector := newOneDriveTestConnector()+ connector.getJSON = oneDriveDeltaStub(`{"value":[{"id":"drive-1"}]}`, `{"value":[]}`)+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: &SyncCheckpoint{}})+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)+ }+}++func TestOneDriveOpenSyncResumeRejectsMissingAnchor(t *testing.T) {+ connector := newOneDriveTestConnector()+ connector.getJSON = oneDriveDeltaStub(`{"value":[{"id":"drive-1"}]}`, `{"value":[+ {"id":"file-1","name":"a.pdf","file":{},"size":1,"eTag":"e1"},+ {"id":"file-3","name":"c.pdf","file":{},"size":1,"eTag":"e3"}+ ]}`)+ checkpoint := &SyncCheckpoint{+ Cursor: `{"drive_id":"drive-1","page_url":"` + connector.deltaURL("drive-1") + `","source_id":"file-2"}`,+ }+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ if _, err := session.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("resume NextBatch err = %v, want ErrSyncResumeInvalid", err)+ }+}++func TestOneDriveOpenPrune(t *testing.T) {+ connector := newOneDriveTestConnector()+ connector.batchSize = 1+ firstURL := connector.deltaURL("drive-1")+ secondURL := "https://graph.microsoft.com/v1.0/next"+ connector.getJSON = func(ctx context.Context, apiURL string, out any) error {+ switch apiURL {+ case connector.graphBaseURL + "/drives":+ return json.Unmarshal([]byte(`{"value":[{"id":"drive-1"}]}`), out)+ case firstURL:+ return json.Unmarshal([]byte(`{"value":[+ {"id":"file-1","name":"a.pdf","file":{},"size":1},+ {"id":"folder","name":"Docs","folder":{}},+ {"id":"deleted","name":"gone.pdf","file":{},"deleted":{"state":"deleted"}},+ {"id":"photo","name":"photo.png","file":{},"size":1}+ ],"@odata.nextLink":"`+secondURL+`"}`), out)+ case secondURL:+ return json.Unmarshal([]byte(`{"value":[{"id":"file-2","name":"b.pdf","file":{},"size":1}]}`), out)+ default:+ t.Fatalf("unexpected prune URL %s", apiURL)+ return nil+ }+ }+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune failed: %v", err)+ }+ var got []string+ for {+ batch, err := session.NextBatch(context.Background())+ if errors.Is(err, io.EOF) {+ break+ }+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ for _, doc := range batch.Documents {+ got = append(got, doc.SourceID)+ }+ }+ if len(got) != 2 || got[0] != "file-1" || got[1] != "file-2" {+ t.Fatalf("prune documents = %v, want file-1 and file-2", got)+ }+}++func TestOneDrivePrunePaginationStall(t *testing.T) {+ connector := newOneDriveTestConnector()+ pageURL := connector.deltaURL("drive-1")+ connector.getJSON = func(ctx context.Context, apiURL string, out any) error {+ if apiURL == connector.graphBaseURL+"/drives" {+ return json.Unmarshal([]byte(`{"value":[{"id":"drive-1"}]}`), out)+ }+ return json.Unmarshal([]byte(`{"value":[{"id":"file-1","name":"a.pdf","file":{},"size":1}],"@odata.nextLink":"`+pageURL+`"}`), out)+ }+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune failed: %v", err)+ }+ if _, err := session.NextBatch(context.Background()); err == nil || !strings.Contains(err.Error(), "did not advance") {+ t.Fatalf("prune NextBatch err = %v, want stalled pagination error", err)+ }+}++func TestOneDriveFetch(t *testing.T) {+ connector := newOneDriveTestConnector()+ connector.sizeThreshold = 5+ ref := FetchReference{Key: `{"drive_id":"drive-1","item_id":"item-1","name":"big.pdf","size":10}`}+ if _, err := connector.Fetch(context.Background(), ref); err == nil || !strings.Contains(err.Error(), "exceeds size threshold") {+ t.Fatalf("oversize Fetch err = %v, want size threshold error", err)+ }++ var fetchedURL string+ connector.getBytes = func(ctx context.Context, apiURL string) ([]byte, error) {+ fetchedURL = apiURL+ if !strings.Contains(apiURL, "/items/item-1/content") {+ t.Fatalf("fetch URL = %q, want item-1 content", apiURL)+ }+ return []byte("hello"), nil+ }+ ref.Key = `{"drive_id":"drive-1","item_id":"item-1","name":"ok.pdf","size":5}`+ blob, err := connector.Fetch(context.Background(), ref)+ if err != nil {+ t.Fatalf("Fetch failed: %v", err)+ }+ if string(blob) != "hello" || fetchedURL == "" {+ t.Fatalf("fetch blob = %q, url = %q", blob, fetchedURL)+ }+}++func TestOneDriveDeltaURL(t *testing.T) {+ connector := newOneDriveTestConnector()+ if got := connector.deltaURL("drive 1"); got != "https://graph.microsoft.com/v1.0/drives/drive%201/root/delta" {+ t.Fatalf("root delta URL = %q", got)+ }+ connector.folderPath = "/Documents/A B"+ if got := connector.deltaURL("drive-1"); got != "https://graph.microsoft.com/v1.0/drives/drive-1/root:/Documents/A%20B:/delta" {+ t.Fatalf("scoped delta URL = %q", got)+ }+}++func newOneDriveTestConnector() *OneDriveConnector {+ return &OneDriveConnector{+ tenantID: "tenant",+ clientID: "client",+ clientSecret: "secret",+ batchSize: onedriveDefaultBatchSize,+ sizeThreshold: onedriveDefaultSizeThreshold,+ graphBaseURL: onedriveGraphBase,+ httpClient: http.DefaultClient,+ now: time.Now,+ }+}++func oneDriveDeltaStub(drivesJSON, deltaJSON string) func(ctx context.Context, apiURL string, out any) error {+ return func(ctx context.Context, apiURL string, out any) error {+ if strings.HasSuffix(apiURL, "/drives") {+ return json.Unmarshal([]byte(drivesJSON), out)+ }+ return json.Unmarshal([]byte(deltaJSON), out)+ }+}++func onedriveMustTimePointer(t *testing.T, value string) *time.Time {+ t.Helper()+ parsed := mustTime(t, value)+ return &parsed+}