RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(go-syncer): add Asana connector with resume and prune (#19049)
mkaaad78527d72cf73
diff --git a/internal/syncer/connector/asana.go b/internal/syncer/connector/asana.gonew file mode 100644index 0000000000..e3a67fb8f9--- /dev/null+++ b/internal/syncer/connector/asana.go@@ -0,0 +1,1101 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "path/filepath"+ "sort"+ "strconv"+ "strings"+ "time"++ "ragflow/internal/utility"+)++const (+ asanaDefaultBatchSize = 2+ asanaDefaultSizeThreshold = 20 * 1024 * 1024+ asanaAPIBaseURL = "https://app.asana.com/api/1.0"+ asanaRequestTimeout = 60 * time.Second+ asanaRetryCount = 4+ asanaRetryBaseDelay = 200 * time.Millisecond+ asanaMaxRetryDelay = 30 * time.Second+ asanaMaxJSONResponseSize = 32 * 1024 * 1024+ asanaPageSize = 100+ asanaTaskOptFields = "gid,name,notes,permalink_url,created_at,modified_at,completed_at,due_on,created_by.name,memberships.project.gid,memberships.project.name"+ asanaProjectOptFields = "gid,name,archived,team.gid,privacy_setting"+ asanaStoryOptFields = "text,created_at,created_by.name,resource_subtype"+ asanaAttachmentOptFields = "gid,name,download_url,size,created_at"+ asanaSourcePrefix = "asana:"+)++// AsanaConnector reads Asana tasks, comments, and attachments.+type AsanaConnector struct {+ workspaceID string+ projectIDs []string+ teamID string+ token string+ batchSize int+ sizeThreshold int64+ apiBaseURL string+ httpClient *http.Client++ doJSON func(ctx context.Context, apiPath string, query url.Values, out any) error+ download func(ctx context.Context, rawURL string, maxSize int64) ([]byte, error)+}++type asanaAPIError struct {+ Status int+ Message string+}++func (e *asanaAPIError) Error() string {+ if e.Status == 0 {+ return e.Message+ }+ return fmt.Sprintf("Asana API returned HTTP %d: %s", e.Status, e.Message)+}++type asanaNextPage struct {+ Offset string `json:"offset"`+}++type asanaListEnvelope struct {+ Data json.RawMessage `json:"data"`+ NextPage *asanaNextPage `json:"next_page"`+}++type asanaObjectEnvelope struct {+ Data json.RawMessage `json:"data"`+}++type asanaWorkspace struct {+ GID string `json:"gid"`+ Name string `json:"name"`+}++type asanaUser struct {+ GID string `json:"gid"`+ Name string `json:"name"`+ Email string `json:"email"`+}++type asanaProject struct {+ GID string `json:"gid"`+ Name string `json:"name"`+ Archived bool `json:"archived"`+ PrivacySetting string `json:"privacy_setting"`+ Team *struct {+ GID string `json:"gid"`+ } `json:"team"`+}++type asanaTaskMembership struct {+ Project asanaProject `json:"project"`+}++type asanaTask struct {+ GID string `json:"gid"`+ Name string `json:"name"`+ Notes string `json:"notes"`+ PermalinkURL string `json:"permalink_url"`+ CreatedAt string `json:"created_at"`+ ModifiedAt string `json:"modified_at"`+ CompletedAt string `json:"completed_at"`+ DueOn string `json:"due_on"`+ CreatedBy asanaUser `json:"created_by"`+ Memberships []asanaTaskMembership `json:"memberships"`+}++type asanaStory struct {+ GID string `json:"gid"`+ ResourceSubtype string `json:"resource_subtype"`+ Text string `json:"text"`+ CreatedAt string `json:"created_at"`+ CreatedBy asanaUser `json:"created_by"`+}++type asanaAttachment struct {+ GID string `json:"gid"`+ Name string `json:"name"`+ DownloadURL string `json:"download_url"`+ Size int64 `json:"size"`+ CreatedAt string `json:"created_at"`+}++type asanaFetchReference struct {+ TaskGID string `json:"task_gid"`+ AttachmentGID string `json:"attachment_gid"`+ Filename string `json:"filename"`+ DownloadURL string `json:"download_url"`+ Size int64 `json:"size"`+}++type asanaSyncCursor struct {+ ProjectGID string `json:"project_gid"`+ PageOffset string `json:"page_offset,omitempty"`+ SourceID string `json:"source_id"`+}++// NewAsanaConnector creates an Asana connector from Python-compatible config.+func NewAsanaConnector(config map[string]any) (*AsanaConnector, error) {+ credentials := configAnyMap(config["credentials"])+ connector := &AsanaConnector{+ workspaceID: strings.TrimSpace(stringConfig(config["asana_workspace_id"])),+ projectIDs: asanaProjectIDList(stringConfig(config["asana_project_ids"])),+ teamID: strings.TrimSpace(stringConfig(config["asana_team_id"])),+ token: strings.TrimSpace(stringConfig(credentials["asana_api_token_secret"])),+ batchSize: configInt(firstNonEmpty(stringConfig(config["sync_batch_size"]), stringConfig(config["batch_size"])), asanaDefaultBatchSize),+ apiBaseURL: asanaAPIBaseURL,+ httpClient: &http.Client{Timeout: asanaRequestTimeout},+ }+ connector.sizeThreshold = int64(configInt(config["size_threshold"], asanaDefaultSizeThreshold))+ if connector.sizeThreshold <= 0 {+ connector.sizeThreshold = asanaDefaultSizeThreshold+ }+ connector.doJSON = connector.doAsanaJSON+ connector.download = connector.downloadAsanaFile+ return connector, nil+}++func asanaProjectIDList(value string) []string {+ parts := strings.Split(value, ",")+ items := make([]string, 0, len(parts))+ for _, part := range parts {+ if trimmed := strings.TrimSpace(part); trimmed != "" {+ items = append(items, trimmed)+ }+ }+ return items+}++// Validate validates Asana settings and credentials.+func (c *AsanaConnector) Validate(ctx context.Context) error {+ if err := c.validateStatic(); err != nil {+ return err+ }+ var envelope asanaObjectEnvelope+ if err := c.doJSON(ctx, "workspaces/"+url.PathEscape(c.workspaceID), url.Values{"opt_fields": {"gid", "name"}}, &envelope); err != nil {+ return classifyAsanaError(err)+ }+ var workspace asanaWorkspace+ if err := json.Unmarshal(envelope.Data, &workspace); err != nil {+ return err+ }+ if workspace.GID == "" {+ return &ConnectorValidationError{Message: "Asana returned an invalid workspace response."}+ }+ return nil+}++// ValidateConnectorSetting validates an unsaved Asana config.+func (c *AsanaConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ candidate, err := NewAsanaConnector(request)+ if err != nil {+ return err+ }+ if c != nil {+ if c.httpClient != nil {+ candidate.httpClient = c.httpClient+ }+ if c.apiBaseURL != "" {+ candidate.apiBaseURL = c.apiBaseURL+ }+ }+ return candidate.Validate(ctx)+}++// OpenSync opens one Asana sync session.+func (c *AsanaConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ if err := c.validateStatic(); err != nil {+ return nil, err+ }+ session := &asanaSyncSession{+ connector: c,+ request: request,+ batchSize: c.effectiveBatchSize(),+ seenTasks: map[string]bool{},+ }+ if err := session.applyResume(request.Resume); err != nil {+ return nil, err+ }+ return session, nil+}++// OpenPrune opens one complete Asana prune snapshot session.+func (c *AsanaConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ if err := c.validateStatic(); err != nil {+ return nil, err+ }+ return &asanaPruneSession{+ connector: c,+ batchSize: c.effectiveBatchSize(),+ seenTasks: map[string]bool{},+ }, nil+}++// Fetch downloads a delayed Asana attachment.+func (c *AsanaConnector) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ var fetch asanaFetchReference+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {+ return nil, err+ }+ if fetch.TaskGID == "" || fetch.AttachmentGID == "" || fetch.DownloadURL == "" {+ return nil, fmt.Errorf("asana fetch reference is incomplete")+ }+ if fetch.Size > c.sizeThreshold {+ return nil, fmt.Errorf("%s exceeds size threshold of %d", firstNonEmpty(fetch.Filename, fetch.AttachmentGID), c.sizeThreshold)+ }+ if c.download != nil {+ return c.download(ctx, fetch.DownloadURL, c.sizeThreshold)+ }+ return c.downloadAsanaFile(ctx, fetch.DownloadURL, c.sizeThreshold)+}++func (c *AsanaConnector) validateStatic() error {+ if c == nil {+ return &ConnectorValidationError{Message: "Asana connector is nil"}+ }+ if c.workspaceID == "" {+ return &ConnectorValidationError{Message: "Asana workspace_id is required"}+ }+ if c.token == "" {+ return &ConnectorMissingCredentialError{Message: "Asana asana_api_token_secret is required"}+ }+ if c.batchSize <= 0 {+ return &ConnectorValidationError{Message: "Asana connector batch_size must be a positive integer"}+ }+ if c.sizeThreshold <= 0 {+ c.sizeThreshold = asanaDefaultSizeThreshold+ }+ if c.apiBaseURL == "" {+ c.apiBaseURL = asanaAPIBaseURL+ }+ if c.httpClient == nil {+ c.httpClient = &http.Client{Timeout: asanaRequestTimeout}+ }+ return nil+}++func (c *AsanaConnector) effectiveBatchSize() int {+ if c.batchSize > 0 {+ return c.batchSize+ }+ return asanaDefaultBatchSize+}++func (c *AsanaConnector) apiURL(apiPath string, query url.Values) string {+ base := strings.TrimRight(c.apiBaseURL, "/") + "/" + strings.TrimPrefix(apiPath, "/")+ if len(query) == 0 {+ return base+ }+ return base + "?" + query.Encode()+}++func (c *AsanaConnector) doAsanaJSON(ctx context.Context, apiPath string, query url.Values, out any) error {+ delay := asanaRetryBaseDelay+ var lastErr error+ for attempt := 1; attempt <= asanaRetryCount; attempt++ {+ requestCtx, cancel := context.WithTimeout(ctx, asanaRequestTimeout)+ request, err := http.NewRequestWithContext(requestCtx, http.MethodGet, c.apiURL(apiPath, query), nil)+ if err != nil {+ cancel()+ return err+ }+ request.Header.Set("Authorization", "Bearer "+c.token)+ request.Header.Set("Accept", "application/json")++ response, err := c.httpClient.Do(request)+ if err != nil {+ cancel()+ lastErr = err+ } else {+ body, readErr := io.ReadAll(io.LimitReader(response.Body, asanaMaxJSONResponseSize+1))+ _ = response.Body.Close()+ cancel()+ if readErr != nil {+ return readErr+ }+ if response.StatusCode >= 400 {+ lastErr = &asanaAPIError{Status: response.StatusCode, Message: strings.TrimSpace(string(body))}+ if !isAsanaRetryable(response.StatusCode) {+ return lastErr+ }+ } else {+ if int64(len(body)) > asanaMaxJSONResponseSize {+ return fmt.Errorf("asana API response exceeds maximum size of %d bytes", asanaMaxJSONResponseSize)+ }+ if err := json.Unmarshal(body, out); err != nil {+ return err+ }+ return nil+ }+ }+ if attempt == asanaRetryCount {+ break+ }+ wait := delay+ if response != nil {+ wait = asanaRetryAfter(response, delay)+ }+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-time.After(wait):+ }+ delay = min(delay*2, asanaMaxRetryDelay)+ }+ if lastErr == nil {+ lastErr = fmt.Errorf("Asana request failed after %d attempts", asanaRetryCount)+ }+ return lastErr+}++func isAsanaRetryable(status int) bool {+ return status == http.StatusTooManyRequests || status == http.StatusRequestTimeout || status >= 500+}++func asanaRetryAfter(response *http.Response, fallback time.Duration) time.Duration {+ value := strings.TrimSpace(response.Header.Get("Retry-After"))+ if seconds, err := strconv.Atoi(value); err == nil && seconds >= 0 {+ wait := time.Duration(seconds) * time.Second+ return min(wait, asanaMaxRetryDelay)+ }+ if retryTime, err := http.ParseTime(value); err == nil {+ wait := time.Until(retryTime)+ if wait < 0 {+ wait = 0+ }+ return min(wait, asanaMaxRetryDelay)+ }+ return fallback+}++func classifyAsanaError(err error) error {+ var apiErr *asanaAPIError+ if !errors.As(err, &apiErr) {+ return err+ }+ switch apiErr.Status {+ case http.StatusUnauthorized:+ return &ConnectorMissingCredentialError{Message: "Asana access token is invalid or expired."}+ case http.StatusForbidden:+ return &ConnectorValidationError{Message: "Asana token does not have permission to read the requested workspace or project."}+ case http.StatusNotFound:+ return &ConnectorValidationError{Message: "Asana workspace or project was not found."}+ case http.StatusTooManyRequests:+ return &ConnectorValidationError{Message: "Asana rate limit exceeded during validation."}+ default:+ return &ConnectorValidationError{Message: fmt.Sprintf("Asana validation failed (HTTP %d): %s", apiErr.Status, apiErr.Message)}+ }+}++func (c *AsanaConnector) listPage(ctx context.Context, apiPath string, query url.Values, offset string, out any) (string, error) {+ query.Set("limit", strconv.Itoa(asanaPageSize))+ if offset != "" {+ query.Set("offset", offset)+ }+ var envelope asanaListEnvelope+ if err := c.doJSON(ctx, apiPath, query, &envelope); err != nil {+ return "", err+ }+ if len(envelope.Data) > 0 {+ if err := json.Unmarshal(envelope.Data, out); err != nil {+ return "", err+ }+ }+ if envelope.NextPage == nil {+ return "", nil+ }+ return envelope.NextPage.Offset, nil+}++func (c *AsanaConnector) selectProjects(ctx context.Context) ([]asanaProject, error) {+ projects := make([]asanaProject, 0)+ if len(c.projectIDs) > 0 {+ for _, projectID := range c.projectIDs {+ project, err := c.getProject(ctx, projectID)+ if err != nil {+ return nil, err+ }+ if c.isProjectProcessable(project) {+ projects = append(projects, project)+ }+ }+ } else {+ query := url.Values{+ "workspace": {c.workspaceID},+ "opt_fields": {asanaProjectOptFields},+ }+ offset := ""+ for {+ var page []asanaProject+ next, err := c.listPage(ctx, "projects", query, offset, &page)+ if err != nil {+ return nil, err+ }+ for _, project := range page {+ if c.isProjectProcessable(project) {+ projects = append(projects, project)+ }+ }+ if next == "" {+ break+ }+ if next == offset {+ return nil, fmt.Errorf("Asana projects pagination did not advance")+ }+ offset = next+ }+ }+ sort.Slice(projects, func(i, j int) bool {+ return projects[i].GID < projects[j].GID+ })+ return projects, nil+}++func (c *AsanaConnector) getProject(ctx context.Context, projectGID string) (asanaProject, error) {+ var envelope asanaObjectEnvelope+ if err := c.doJSON(ctx, "projects/"+url.PathEscape(projectGID), url.Values{"opt_fields": {asanaProjectOptFields}}, &envelope); err != nil {+ return asanaProject{}, err+ }+ var project asanaProject+ if err := json.Unmarshal(envelope.Data, &project); err != nil {+ return asanaProject{}, err+ }+ return project, nil+}++func (c *AsanaConnector) isProjectProcessable(project asanaProject) bool {+ if project.GID == "" || project.Archived {+ return false+ }+ if project.Team == nil || project.Team.GID == "" {+ return false+ }+ if project.PrivacySetting == "private" && c.teamID != "" && project.Team.GID != c.teamID {+ return false+ }+ return true+}++func (c *AsanaConnector) listTasksPage(ctx context.Context, projectGID, offset string, since *time.Time) ([]asanaTask, string, error) {+ query := url.Values{+ "project": {projectGID},+ "opt_fields": {asanaTaskOptFields},+ }+ if since != nil && !since.IsZero() {+ query.Set("modified_since", since.UTC().Format(time.RFC3339Nano))+ }+ var tasks []asanaTask+ next, err := c.listPage(ctx, "tasks", query, offset, &tasks)+ return tasks, next, err+}++func (c *AsanaConnector) listStories(ctx context.Context, taskGID string) ([]asanaStory, error) {+ query := url.Values{"opt_fields": {asanaStoryOptFields}}+ offset := ""+ stories := make([]asanaStory, 0)+ for {+ var page []asanaStory+ next, err := c.listPage(ctx, "tasks/"+url.PathEscape(taskGID)+"/stories", query, offset, &page)+ if err != nil {+ return nil, err+ }+ stories = append(stories, page...)+ if next == "" {+ break+ }+ if next == offset {+ return nil, fmt.Errorf("Asana stories pagination did not advance")+ }+ offset = next+ }+ sort.Slice(stories, func(i, j int) bool {+ left := parseOutlookTime(stories[i].CreatedAt)+ right := parseOutlookTime(stories[j].CreatedAt)+ if !left.Equal(right) {+ return left.Before(right)+ }+ return stories[i].GID < stories[j].GID+ })+ return stories, nil+}++func (c *AsanaConnector) listAttachments(ctx context.Context, taskGID string) ([]asanaAttachment, error) {+ query := url.Values{+ "parent": {taskGID},+ "opt_fields": {asanaAttachmentOptFields},+ }+ offset := ""+ attachments := make([]asanaAttachment, 0)+ for {+ var page []asanaAttachment+ next, err := c.listPage(ctx, "attachments", query, offset, &page)+ if err != nil {+ return nil, err+ }+ attachments = append(attachments, page...)+ if next == "" {+ break+ }+ if next == offset {+ return nil, fmt.Errorf("Asana attachments pagination did not advance")+ }+ offset = next+ }+ sort.Slice(attachments, func(i, j int) bool {+ return attachments[i].GID < attachments[j].GID+ })+ return attachments, nil+}++func (c *AsanaConnector) taskDocument(task asanaTask, project asanaProject, comments []asanaStory) (SourceDocument, bool) {+ if strings.TrimSpace(task.GID) == "" {+ return SourceDocument{}, false+ }+ updatedAt := parseOutlookTime(task.ModifiedAt)+ if updatedAt.IsZero() {+ updatedAt = time.Now().UTC()+ }+ blob := []byte(asanaTaskBlob(task, project, comments))+ name := strings.TrimSpace(task.Name)+ if name == "" {+ name = "Untitled task " + task.GID+ }+ return SourceDocument{+ SourceID: asanaTaskSourceID(task.GID),+ SemanticIdentifier: name,+ Extension: ".md",+ Blob: blob,+ UpdatedAt: updatedAt,+ SizeBytes: int64(len(blob)),+ Metadata: map[string]any{+ "task_gid": task.GID,+ "project": project.Name,+ "project_id": project.GID,+ "url": task.PermalinkURL,+ },+ Fingerprint: contentFingerprint(blob),+ }, true+}++func asanaTaskSourceID(taskGID string) string {+ return asanaSourcePrefix + taskGID+}++func asanaAttachmentSourceID(taskGID, attachmentGID string) string {+ return asanaTaskSourceID(taskGID) + ":" + attachmentGID+}++func asanaTaskBlob(task asanaTask, project asanaProject, comments []asanaStory) string {+ var builder strings.Builder+ builder.WriteString(strings.TrimSpace(firstNonEmpty(task.Name, "Untitled task")))+ builder.WriteString("\n\n")+ if notes := strings.TrimSpace(task.Notes); notes != "" {+ builder.WriteString(notes)+ builder.WriteString("\n\n")+ }+ if project.Name != "" {+ builder.WriteString("Project: ")+ builder.WriteString(project.Name)+ builder.WriteString("\n")+ }+ if task.PermalinkURL != "" {+ builder.WriteString("URL: ")+ builder.WriteString(task.PermalinkURL)+ builder.WriteString("\n")+ }+ if task.CreatedBy.Name != "" {+ builder.WriteString("Created by: ")+ builder.WriteString(task.CreatedBy.Name)+ builder.WriteString("\n")+ }+ if created := parseOutlookTime(task.CreatedAt); !created.IsZero() {+ builder.WriteString("Created: ")+ builder.WriteString(created.Format("2006-01-02"))+ builder.WriteString("\n")+ }+ if due := strings.TrimSpace(task.DueOn); due != "" {+ builder.WriteString("Due date: ")+ builder.WriteString(due)+ builder.WriteString("\n")+ }+ if completed := parseOutlookTime(task.CompletedAt); !completed.IsZero() {+ builder.WriteString("Completed on: ")+ builder.WriteString(completed.Format("2006-01-02"))+ builder.WriteString("\n")+ }++ var taskComments []asanaStory+ for _, story := range comments {+ if story.ResourceSubtype != "comment_added" || strings.TrimSpace(story.Text) == "" {+ continue+ }+ taskComments = append(taskComments, story)+ }+ if len(taskComments) > 0 {+ builder.WriteString("\n## Comments\n\n")+ for _, comment := range taskComments {+ if comment.CreatedBy.Name != "" {+ builder.WriteString("Comment by ")+ builder.WriteString(comment.CreatedBy.Name)+ builder.WriteString(": ")+ } else {+ builder.WriteString("Comment: ")+ }+ builder.WriteString(strings.TrimSpace(comment.Text))+ builder.WriteString("\n\n")+ }+ }+ return strings.TrimSpace(builder.String()) + "\n"+}++func (c *AsanaConnector) attachmentDocument(task asanaTask, project asanaProject, attachment asanaAttachment) (SourceDocument, bool) {+ if task.GID == "" || attachment.GID == "" || attachment.DownloadURL == "" || attachment.Name == "" {+ return SourceDocument{}, false+ }+ if attachment.Size < 0 || (attachment.Size > 0 && attachment.Size > c.sizeThreshold) {+ return SourceDocument{}, false+ }+ updatedAt := parseOutlookTime(task.ModifiedAt)+ if updatedAt.IsZero() {+ updatedAt = time.Now().UTC()+ }+ fetch, _ := json.Marshal(asanaFetchReference{+ TaskGID: task.GID,+ AttachmentGID: attachment.GID,+ Filename: attachment.Name,+ DownloadURL: attachment.DownloadURL,+ Size: attachment.Size,+ })+ return SourceDocument{+ SourceID: asanaAttachmentSourceID(task.GID, attachment.GID),+ SemanticIdentifier: attachment.Name,+ Extension: strings.ToLower(filepath.Ext(attachment.Name)),+ FetchRef: &FetchReference{Key: string(fetch), SizeHint: attachment.Size},+ UpdatedAt: updatedAt,+ SizeBytes: attachment.Size,+ Metadata: map[string]any{+ "task_gid": task.GID,+ "attachment_gid": attachment.GID,+ "project": project.Name,+ "project_id": project.GID,+ "filename": attachment.Name,+ "url": attachment.DownloadURL,+ "size": attachment.Size,+ },+ Fingerprint: asanaAttachmentFingerprint(task.GID, attachment),+ }, true+}++func asanaAttachmentFingerprint(taskGID string, attachment asanaAttachment) string {+ return stableFingerprint(map[string]any{+ "task_gid": taskGID,+ "attachment_gid": attachment.GID,+ "filename": attachment.Name,+ "size": attachment.Size,+ "created_at": attachment.CreatedAt,+ })+}++func includeAsanaDocument(request SyncRequest, document SourceDocument) bool {+ if request.FromBeginning {+ return true+ }+ if len(request.Fingerprints) > 0 {+ stored, ok := request.Fingerprints[document.SourceID]+ return !ok || stored == "" || stored != document.Fingerprint+ }+ return !beforeOrAtWindowStart(document.UpdatedAt, request.WindowStart) && !afterWindowEnd(document.UpdatedAt, request.WindowEnd)+}++func (c *AsanaConnector) downloadAsanaFile(ctx context.Context, rawURL string, maxSize int64) ([]byte, error) {+ data, _, _, err := utility.FetchRemoteFileSafely(ctx, rawURL, maxSize)+ return data, err+}++// asanaSyncSession streams Asana task and attachment documents for one window.+type asanaSyncSession struct {+ connector *AsanaConnector+ request SyncRequest+ batchSize int+ projects []asanaProject+ projectIdx int+ pageOffset string+ currentPageOffset string+ taskBuffer []asanaTask+ taskIdx int+ taskPageDone bool+ current asanaProject+ documents []asanaBufferedDocument+ docIdx int+ seenTasks map[string]bool+ projectsOK bool+ done bool++ resumeProjectGID string+ resumePageOffset string+ resumeSourceID string+ resumeMatched bool+}++type asanaBufferedDocument struct {+ document SourceDocument+ checkpoint *SyncCheckpoint+}++func (s *asanaSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ documents := make([]SourceDocument, 0, s.batchSize)+ var checkpoint *SyncCheckpoint++ for len(documents) < s.batchSize {+ if s.docIdx < len(s.documents) {+ buffered := s.documents[s.docIdx]+ s.docIdx+++ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ continue+ }+ if s.done {+ if s.resumeSourceID != "" && !s.resumeMatched {+ return SyncBatch{}, fmt.Errorf("Asana resume anchor %q was not found in the current listing: %w", s.resumeSourceID, ErrSyncResumeInvalid)+ }+ if len(documents) == 0 {+ return SyncBatch{}, io.EOF+ }+ break+ }+ if !s.projectsOK {+ if err := s.loadProjects(ctx); err != nil {+ return SyncBatch{}, err+ }+ if len(s.projects) == 0 {+ s.done = true+ continue+ }+ }+ if s.taskIdx >= len(s.taskBuffer) {+ if s.projectIdx >= len(s.projects) {+ s.done = true+ continue+ }+ if s.taskPageDone {+ s.projectIdx+++ s.pageOffset = ""+ s.taskBuffer = nil+ s.taskPageDone = false+ continue+ }+ nextPage, err := s.loadTaskPage(ctx)+ if err != nil {+ return SyncBatch{}, err+ }+ if len(s.taskBuffer) == 0 {+ if nextPage == "" {+ s.projectIdx+++ s.pageOffset = ""+ s.taskBuffer = nil+ s.taskPageDone = false+ }+ continue+ }+ }++ task := s.taskBuffer[s.taskIdx]+ s.taskIdx+++ if s.seenTasks[task.GID] {+ continue+ }+ s.seenTasks[task.GID] = true+ docs, err := s.taskDocuments(ctx, task)+ if err != nil {+ return SyncBatch{}, err+ }+ docs, err = s.filterResumedDocuments(docs)+ if err != nil {+ return SyncBatch{}, err+ }+ s.documents = docs+ s.docIdx = 0+ }++ if len(documents) == 0 {+ return SyncBatch{}, io.EOF+ }+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil+}++func (s *asanaSyncSession) Close() error {+ return nil+}++// Fetch downloads a delayed Asana attachment for this sync session.+func (s *asanaSyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ return s.connector.Fetch(ctx, ref)+}++func (s *asanaSyncSession) loadProjects(ctx context.Context) error {+ projects, err := s.connector.selectProjects(ctx)+ if err != nil {+ return err+ }+ s.projects = projects+ s.projectsOK = true+ if s.resumeProjectGID != "" {+ found := false+ for index, project := range projects {+ if project.GID == s.resumeProjectGID {+ s.projectIdx = index+ found = true+ break+ }+ }+ if !found {+ return fmt.Errorf("Asana resume project %q was not found in the current listing: %w", s.resumeProjectGID, ErrSyncResumeInvalid)+ }+ s.pageOffset = s.resumePageOffset+ }+ return nil+}++func (s *asanaSyncSession) loadTaskPage(ctx context.Context) (string, error) {+ project := s.projects[s.projectIdx]+ s.current = project+ s.currentPageOffset = s.pageOffset+ tasks, next, err := s.connector.listTasksPage(ctx, project.GID, s.pageOffset, asanaSyncSince(s.request))+ if err != nil {+ return "", err+ }+ if next != "" && next == s.pageOffset {+ return "", fmt.Errorf("Asana tasks pagination did not advance for project %s", project.GID)+ }+ s.taskBuffer = tasks+ s.taskIdx = 0+ s.pageOffset = next+ s.taskPageDone = next == ""+ return next, nil+}++func asanaSyncSince(request SyncRequest) *time.Time {+ if request.FromBeginning {+ return nil+ }+ return request.WindowStart+}++func (s *asanaSyncSession) taskDocuments(ctx context.Context, task asanaTask) ([]asanaBufferedDocument, error) {+ stories, err := s.connector.listStories(ctx, task.GID)+ if err != nil {+ return nil, err+ }+ taskDoc, ok := s.connector.taskDocument(task, s.current, stories)+ if !ok {+ return nil, nil+ }+ documents := []asanaBufferedDocument{}+ if includeAsanaDocument(s.request, taskDoc) {+ documents = append(documents, asanaBufferedDocument{+ document: taskDoc,+ checkpoint: s.checkpoint(taskDoc),+ })+ }++ attachments, err := s.connector.listAttachments(ctx, task.GID)+ if err != nil {+ return nil, err+ }+ for _, attachment := range attachments {+ doc, ok := s.connector.attachmentDocument(task, s.current, attachment)+ if !ok {+ continue+ }+ if includeAsanaDocument(s.request, doc) {+ documents = append(documents, asanaBufferedDocument{+ document: doc,+ checkpoint: s.checkpoint(doc),+ })+ }+ }+ return documents, nil+}++func (s *asanaSyncSession) checkpoint(document SourceDocument) *SyncCheckpoint {+ cursor, err := json.Marshal(asanaSyncCursor{+ ProjectGID: s.current.GID,+ PageOffset: s.currentPageOffset,+ SourceID: document.SourceID,+ })+ if err != nil {+ return nil+ }+ updatedAt := document.UpdatedAt+ return &SyncCheckpoint{Cursor: string(cursor), SourceID: document.SourceID, UpdatedAt: &updatedAt}+}++func (s *asanaSyncSession) applyResume(checkpoint *SyncCheckpoint) error {+ if checkpoint == nil {+ return nil+ }+ if checkpoint.Cursor == "" {+ return fmt.Errorf("Asana sync cursor is missing: %w", ErrSyncResumeInvalid)+ }+ var cursor asanaSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ return fmt.Errorf("Asana sync cursor is invalid: %w", ErrSyncResumeInvalid)+ }+ sourceID := firstNonEmpty(cursor.SourceID, checkpoint.SourceID)+ if sourceID == "" || !strings.HasPrefix(sourceID, asanaSourcePrefix) {+ return fmt.Errorf("Asana sync checkpoint has no source anchor: %w", ErrSyncResumeInvalid)+ }+ if cursor.ProjectGID == "" {+ return fmt.Errorf("Asana sync checkpoint has no project anchor: %w", ErrSyncResumeInvalid)+ }+ s.resumeProjectGID = cursor.ProjectGID+ s.resumePageOffset = cursor.PageOffset+ s.resumeSourceID = sourceID+ return nil+}++func (s *asanaSyncSession) filterResumedDocuments(candidates []asanaBufferedDocument) ([]asanaBufferedDocument, error) {+ if s.resumeSourceID == "" || s.resumeMatched {+ return candidates, nil+ }+ for index, candidate := range candidates {+ if candidate.document.SourceID == s.resumeSourceID {+ s.resumeMatched = true+ s.resumeSourceID = ""+ return candidates[index+1:], nil+ }+ }+ return nil, nil+}++// asanaPruneSession streams the complete Asana slim snapshot.+type asanaPruneSession struct {+ connector *AsanaConnector+ batchSize int+ projects []asanaProject+ projectIdx int+ pageOffset string+ taskBuffer []asanaTask+ taskIdx int+ taskPageDone bool+ seenTasks map[string]bool+ projectsOK bool+ done bool+}++func (s *asanaPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ documents := make([]SlimDocument, 0, s.batchSize)+ for len(documents) < s.batchSize {+ if s.done {+ if len(documents) == 0 {+ return PruneBatch{}, io.EOF+ }+ break+ }+ if !s.projectsOK {+ projects, err := s.connector.selectProjects(ctx)+ if err != nil {+ return PruneBatch{}, err+ }+ s.projects = projects+ s.projectsOK = true+ if len(projects) == 0 {+ s.done = true+ continue+ }+ }+ if s.taskIdx >= len(s.taskBuffer) {+ if s.projectIdx >= len(s.projects) {+ s.done = true+ continue+ }+ if s.taskPageDone {+ s.projectIdx+++ s.pageOffset = ""+ s.taskBuffer = nil+ s.taskPageDone = false+ continue+ }+ next, err := s.loadTaskPage(ctx)+ if err != nil {+ return PruneBatch{}, err+ }+ if len(s.taskBuffer) == 0 {+ if next == "" {+ s.projectIdx+++ s.pageOffset = ""+ s.taskBuffer = nil+ s.taskPageDone = false+ }+ continue+ }+ }++ task := s.taskBuffer[s.taskIdx]+ s.taskIdx+++ if task.GID == "" || s.seenTasks[task.GID] {+ continue+ }+ s.seenTasks[task.GID] = true+ documents = append(documents, SlimDocument{SourceID: asanaTaskSourceID(task.GID)})+ attachments, err := s.connector.listAttachments(ctx, task.GID)+ if err != nil {+ return PruneBatch{}, err+ }+ for _, attachment := range attachments {+ if attachment.GID != "" {+ documents = append(documents, SlimDocument{SourceID: asanaAttachmentSourceID(task.GID, attachment.GID)})+ }+ }+ }+ if len(documents) == 0 {+ return PruneBatch{}, io.EOF+ }+ return PruneBatch{Documents: documents}, nil+}++func (s *asanaPruneSession) Close() error {+ return nil+}++func (s *asanaPruneSession) loadTaskPage(ctx context.Context) (string, error) {+ project := s.projects[s.projectIdx]+ tasks, next, err := s.connector.listTasksPage(ctx, project.GID, s.pageOffset, nil)+ if err != nil {+ return "", err+ }+ if next != "" && next == s.pageOffset {+ return "", fmt.Errorf("Asana prune tasks pagination did not advance for project %s", project.GID)+ }+ s.taskBuffer = tasks+ s.taskIdx = 0+ s.pageOffset = next+ s.taskPageDone = next == ""+ return next, nil+}diff --git a/internal/syncer/connector/asana_test.go b/internal/syncer/connector/asana_test.gonew file mode 100644index 0000000000..a003fa3962--- /dev/null+++ b/internal/syncer/connector/asana_test.go@@ -0,0 +1,614 @@+package connector++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/http/httptest"+ "net/url"+ "strings"+ "testing"+)++func asanaTestConnector(t *testing.T, doJSON func(ctx context.Context, apiPath string, query url.Values, out any) error) *AsanaConnector {+ t.Helper()+ connector, err := NewAsanaConnector(map[string]any{+ "asana_workspace_id": "workspace_1",+ "asana_project_ids": "",+ "asana_team_id": "team_1",+ "batch_size": 2,+ "credentials": map[string]any{+ "asana_api_token_secret": "token",+ },+ })+ if err != nil {+ t.Fatalf("NewAsanaConnector failed: %v", err)+ }+ connector.doJSON = doJSON+ connector.download = func(ctx context.Context, rawURL string, maxSize int64) ([]byte, error) {+ return []byte("attachment body"), nil+ }+ return connector+}++func asanaSetEnvelope(out any, data any, nextPage string) {+ switch typed := out.(type) {+ case *asanaListEnvelope:+ typed.Data, _ = json.Marshal(data)+ if nextPage != "" {+ typed.NextPage = &asanaNextPage{Offset: nextPage}+ }+ case *asanaObjectEnvelope:+ typed.Data, _ = json.Marshal(data)+ default:+ panic(fmt.Sprintf("unexpected Asana response target %T", out))+ }+}++func asanaFixtureDoJSON(projects []asanaProject, tasks []asanaTask, stories []asanaStory, attachments []asanaAttachment) func(ctx context.Context, apiPath string, query url.Values, out any) error {+ return func(ctx context.Context, apiPath string, query url.Values, out any) error {+ switch apiPath {+ case "projects":+ asanaSetEnvelope(out, projects, "")+ case "tasks":+ asanaSetEnvelope(out, tasks, "")+ case "attachments":+ asanaSetEnvelope(out, attachments, "")+ default:+ if strings.HasPrefix(apiPath, "tasks/") && strings.HasSuffix(apiPath, "/stories") {+ asanaSetEnvelope(out, stories, "")+ return nil+ }+ return fmt.Errorf("unexpected Asana API path %q", apiPath)+ }+ return nil+ }+}++func asanaTestProject(gid, name, teamID string) asanaProject {+ return asanaProject{+ GID: gid,+ Name: name,+ Team: &struct {+ GID string `json:"gid"`+ }{GID: teamID},+ }+}++func asanaTestTask(gid, name, modifiedAt string) asanaTask {+ return asanaTask{+ GID: gid,+ Name: name,+ Notes: "Task notes",+ PermalinkURL: "https://app.asana.com/0/task/" + gid,+ CreatedAt: "2026-01-01T00:00:00Z",+ ModifiedAt: modifiedAt,+ CreatedBy: asanaUser{Name: "Alice"},+ }+}++func TestAsanaConnectorConfigParsing(t *testing.T) {+ connector, err := NewAsanaConnector(map[string]any{+ "asana_workspace_id": "workspace_1",+ "asana_project_ids": "project_1, project_2",+ "asana_team_id": "team_1",+ "sync_batch_size": 7,+ "size_threshold": 12345,+ "credentials": map[string]any{+ "asana_api_token_secret": "token",+ },+ })+ if err != nil {+ t.Fatalf("NewAsanaConnector failed: %v", err)+ }+ if connector.workspaceID != "workspace_1" {+ t.Fatalf("workspace id = %q", connector.workspaceID)+ }+ if len(connector.projectIDs) != 2 || connector.projectIDs[0] != "project_1" || connector.projectIDs[1] != "project_2" {+ t.Fatalf("project ids = %#v", connector.projectIDs)+ }+ if connector.batchSize != 7 {+ t.Fatalf("batch size = %d, want 7", connector.batchSize)+ }+ if connector.sizeThreshold != 12345 {+ t.Fatalf("size threshold = %d, want 12345", connector.sizeThreshold)+ }+}++func TestAsanaConnectorValidate(t *testing.T) {+ connector := asanaTestConnector(t, func(ctx context.Context, apiPath string, query url.Values, out any) error {+ if apiPath != "workspaces/workspace_1" {+ return fmt.Errorf("unexpected validation path %q", apiPath)+ }+ asanaSetEnvelope(out, asanaWorkspace{GID: "workspace_1", Name: "Workspace"}, "")+ return nil+ })+ if err := connector.Validate(context.Background()); err != nil {+ t.Fatalf("Validate failed: %v", err)+ }+}++func TestAsanaConnectorValidateClassifiesUnauthorized(t *testing.T) {+ connector := asanaTestConnector(t, func(ctx context.Context, apiPath string, query url.Values, out any) error {+ return &asanaAPIError{Status: 401, Message: "invalid token"}+ })+ err := connector.Validate(context.Background())+ var missing *ConnectorMissingCredentialError+ if !errors.As(err, &missing) {+ t.Fatalf("Validate err = %T %v, want ConnectorMissingCredentialError", err, err)+ }+}++func TestAsanaConnectorValidateStatic(t *testing.T) {+ cases := []struct {+ name string+ conn *AsanaConnector+ valid bool+ }{+ {name: "missing workspace", conn: &AsanaConnector{token: "token", batchSize: 2}},+ {name: "missing token", conn: &AsanaConnector{workspaceID: "workspace_1", batchSize: 2}},+ {name: "invalid batch", conn: &AsanaConnector{workspaceID: "workspace_1", token: "token", batchSize: 0}},+ {name: "valid", conn: &AsanaConnector{workspaceID: "workspace_1", token: "token", batchSize: 2}, valid: true},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ err := tc.conn.validateStatic()+ if tc.valid && err != nil {+ t.Fatalf("validateStatic err = %v", err)+ }+ if !tc.valid && err == nil {+ t.Fatalf("validateStatic unexpectedly passed")+ }+ })+ }+}++func TestAsanaValidateConnectorSettingUsesRequest(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/workspaces/request_workspace" {+ http.Error(w, "unexpected path", http.StatusNotFound)+ return+ }+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"data":{"gid":"request_workspace","name":"Request Workspace"}}`))+ }))+ defer server.Close()++ receiver, err := NewAsanaConnector(map[string]any{+ "asana_workspace_id": "receiver_workspace",+ "credentials": map[string]any{+ "asana_api_token_secret": "receiver_token",+ },+ })+ if err != nil {+ t.Fatalf("NewAsanaConnector failed: %v", err)+ }+ receiver.apiBaseURL = server.URL+ receiver.httpClient = server.Client()+ request := map[string]any{+ "asana_workspace_id": "request_workspace",+ "credentials": map[string]any{+ "asana_api_token_secret": "request_token",+ },+ }+ if err := receiver.ValidateConnectorSetting(context.Background(), request); err != nil {+ t.Fatalf("ValidateConnectorSetting failed: %v", err)+ }+}++func TestAsanaSelectProjectsFilters(t *testing.T) {+ projects := []asanaProject{+ asanaTestProject("p1", "Good", "team_1"),+ {GID: "p2", Name: "Archived", Archived: true, Team: &struct {+ GID string `json:"gid"`+ }{GID: "team_1"}},+ {GID: "p3", Name: "No team"},+ {GID: "p4", Name: "Private", PrivacySetting: "private", Team: &struct {+ GID string `json:"gid"`+ }{GID: "team_other"}},+ }+ connector := asanaTestConnector(t, func(ctx context.Context, apiPath string, query url.Values, out any) error {+ if apiPath != "projects" {+ return fmt.Errorf("unexpected path %q", apiPath)+ }+ asanaSetEnvelope(out, projects, "")+ return nil+ })+ got, err := connector.selectProjects(context.Background())+ if err != nil {+ t.Fatalf("selectProjects failed: %v", err)+ }+ if len(got) != 1 || got[0].GID != "p1" {+ t.Fatalf("projects = %#v, want p1 only", got)+ }+}++func TestAsanaOpenSyncTaskCommentsAndAttachmentFetch(t *testing.T) {+ projects := []asanaProject{asanaTestProject("p1", "Project One", "team_1")}+ tasks := []asanaTask{asanaTestTask("t1", "Task One", "2026-01-02T00:00:00Z")}+ stories := []asanaStory{{+ GID: "s1",+ ResourceSubtype: "comment_added",+ Text: "Looks good",+ CreatedAt: "2026-01-01T01:00:00Z",+ CreatedBy: asanaUser{Name: "Bob"},+ }}+ attachments := []asanaAttachment{{+ GID: "a1",+ Name: "plan.pdf",+ DownloadURL: "https://example.com/plan.pdf",+ Size: 123,+ }}+ connector := asanaTestConnector(t, asanaFixtureDoJSON(projects, tasks, stories, attachments))+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: mustTime(t, "2026-01-03T00:00:00Z")})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2", len(batch.Documents))+ }+ taskDoc := batch.Documents[0]+ if taskDoc.SourceID != "asana:t1" || taskDoc.Extension != ".md" {+ t.Fatalf("task doc = %#v", taskDoc)+ }+ body := string(taskDoc.Blob)+ for _, want := range []string{"Task One", "Task notes", "## Comments", "Comment by Bob: Looks good"} {+ if !strings.Contains(body, want) {+ t.Fatalf("task body missing %q:\n%s", want, body)+ }+ }+ attachmentDoc := batch.Documents[1]+ if attachmentDoc.SourceID != "asana:t1:a1" || attachmentDoc.Extension != ".pdf" || attachmentDoc.FetchRef == nil {+ t.Fatalf("attachment doc = %#v", attachmentDoc)+ }+ fetcher, ok := session.(Fetcher)+ if !ok {+ t.Fatalf("session does not implement Fetcher")+ }+ blob, err := fetcher.Fetch(context.Background(), *attachmentDoc.FetchRef)+ if err != nil {+ t.Fatalf("Fetch failed: %v", err)+ }+ if string(blob) != "attachment body" {+ t.Fatalf("fetched blob = %q", blob)+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("final NextBatch err = %v, want io.EOF", err)+ }+}++func TestAsanaOpenSyncSkipsOversizedAttachment(t *testing.T) {+ projects := []asanaProject{asanaTestProject("p1", "Project One", "team_1")}+ tasks := []asanaTask{asanaTestTask("t1", "Task One", "2026-01-02T00:00:00Z")}+ attachments := []asanaAttachment{{+ GID: "a1",+ Name: "large.bin",+ DownloadURL: "https://example.com/large.bin",+ Size: 1024,+ }}+ connector := asanaTestConnector(t, asanaFixtureDoJSON(projects, tasks, nil, attachments))+ connector.sizeThreshold = 100+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "asana:t1" {+ t.Fatalf("documents = %#v, want task only", batch.Documents)+ }+}++func TestAsanaOpenSyncAdvancesAcrossProjects(t *testing.T) {+ projects := []asanaProject{+ asanaTestProject("p1", "Project One", "team_1"),+ asanaTestProject("p2", "Project Two", "team_1"),+ }+ connector := asanaTestConnector(t, func(ctx context.Context, apiPath string, query url.Values, out any) error {+ switch apiPath {+ case "projects":+ asanaSetEnvelope(out, projects, "")+ case "tasks":+ if query.Get("project") == "p1" {+ if query.Get("offset") == "" {+ asanaSetEnvelope(out, []asanaTask{}, "empty_page")+ } else {+ asanaSetEnvelope(out, []asanaTask{}, "")+ }+ } else {+ asanaSetEnvelope(out, []asanaTask{asanaTestTask("t2", "Task Two", "2026-01-02T00:00:00Z")}, "")+ }+ case "attachments":+ asanaSetEnvelope(out, []asanaAttachment{}, "")+ default:+ if strings.HasPrefix(apiPath, "tasks/") && strings.HasSuffix(apiPath, "/stories") {+ asanaSetEnvelope(out, []asanaStory{}, "")+ return nil+ }+ return fmt.Errorf("unexpected path %q", apiPath)+ }+ return nil+ })+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "asana:t2" {+ t.Fatalf("documents = %#v, want t2 only", batch.Documents)+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("final NextBatch = %v, want io.EOF", err)+ }+}++func TestAsanaIncludeDocumentWindowAndFingerprint(t *testing.T) {+ start := mustTime(t, "2026-01-02T00:00:00Z")+ end := mustTime(t, "2026-01-04T00:00:00Z")+ doc := SourceDocument{SourceID: "asana:t1", UpdatedAt: mustTime(t, "2026-01-03T00:00:00Z"), Fingerprint: "fp"}+ if !includeAsanaDocument(SyncRequest{WindowStart: &start, WindowEnd: end}, doc) {+ t.Fatalf("window doc should be included")+ }+ oldDoc := doc+ oldDoc.UpdatedAt = mustTime(t, "2026-01-01T00:00:00Z")+ if includeAsanaDocument(SyncRequest{WindowStart: &start, WindowEnd: end}, oldDoc) {+ t.Fatalf("old window doc should be excluded")+ }+ request := SyncRequest{Fingerprints: map[string]string{"asana:t1": "fp"}}+ if includeAsanaDocument(request, doc) {+ t.Fatalf("unchanged fingerprint doc should be excluded")+ }+ request.Fingerprints["asana:t1"] = "changed"+ if !includeAsanaDocument(request, doc) {+ t.Fatalf("changed fingerprint doc should be included")+ }+}++func TestAsanaOpenSyncResumeContinuesAfterAnchor(t *testing.T) {+ projects := []asanaProject{asanaTestProject("p1", "Project One", "team_1")}+ connector := asanaTestConnector(t, func(ctx context.Context, apiPath string, query url.Values, out any) error {+ switch apiPath {+ case "projects":+ asanaSetEnvelope(out, projects, "")+ case "tasks":+ if query.Get("offset") == "" {+ asanaSetEnvelope(out, []asanaTask{asanaTestTask("t1", "Task One", "2026-01-02T00:00:00Z")}, "page_2")+ } else {+ asanaSetEnvelope(out, []asanaTask{asanaTestTask("t2", "Task Two", "2026-01-02T01:00:00Z")}, "")+ }+ case "attachments":+ asanaSetEnvelope(out, []asanaAttachment{}, "")+ default:+ if strings.HasPrefix(apiPath, "tasks/") && strings.HasSuffix(apiPath, "/stories") {+ asanaSetEnvelope(out, []asanaStory{}, "")+ return nil+ }+ return fmt.Errorf("unexpected path %q", apiPath)+ }+ return nil+ })+ connector.batchSize = 1+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: mustTime(t, "2026-01-03T00:00:00Z")})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first NextBatch failed: %v", err)+ }+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "asana:t1" || first.Checkpoint == nil {+ t.Fatalf("first batch = %#v", first)+ }++ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: mustTime(t, "2026-01-03T00:00:00Z"), Resume: first.Checkpoint})+ if err != nil {+ t.Fatalf("resumed OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resumed NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "asana:t2" {+ t.Fatalf("resumed documents = %#v, want t2", second.Documents)+ }+ if _, err := resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("resumed final NextBatch = %v, want io.EOF", err)+ }+}++func TestAsanaOpenSyncResumeRejectsInvalidCheckpoint(t *testing.T) {+ connector := asanaTestConnector(t, asanaFixtureDoJSON(nil, nil, nil, nil))+ cases := map[string]*SyncCheckpoint{+ "missing": {},+ "malformed": {Cursor: "not-json"},+ "foreign": {Cursor: `{"project_gid":"p1","source_id":"other:t1"}`},+ "no-anchor": {Cursor: `{"project_gid":"p1"}`},+ }+ for name, checkpoint := range cases {+ t.Run(name, func(t *testing.T) {+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: checkpoint})+ if session != nil || err == nil || !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("resume OpenSync = session %v, err %v, want ErrSyncResumeInvalid", session, err)+ }+ })+ }+}++func TestAsanaOpenSyncResumeRejectsMissingAnchor(t *testing.T) {+ projects := []asanaProject{asanaTestProject("p1", "Project One", "team_1")}+ tasks := []asanaTask{asanaTestTask("t1", "Task One", "2026-01-02T00:00:00Z")}+ connector := asanaTestConnector(t, asanaFixtureDoJSON(projects, tasks, nil, nil))+ cursor, err := json.Marshal(asanaSyncCursor{ProjectGID: "p1", SourceID: "asana:missing"})+ if err != nil {+ t.Fatalf("marshal cursor: %v", err)+ }+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: &SyncCheckpoint{Cursor: string(cursor), SourceID: "asana:missing"}})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ if _, err := session.NextBatch(context.Background()); err == nil || !errors.Is(err, ErrSyncResumeInvalid) {+ t.Fatalf("resume NextBatch err = %v, want ErrSyncResumeInvalid", err)+ }+}++func TestAsanaOpenPruneStreamsAndDeduplicates(t *testing.T) {+ projects := []asanaProject{+ asanaTestProject("p1", "Project One", "team_1"),+ asanaTestProject("p2", "Project Two", "team_1"),+ }+ tasks := []asanaTask{+ asanaTestTask("t1", "Task One", "2026-01-02T00:00:00Z"),+ asanaTestTask("t1", "Task One Duplicate", "2026-01-02T00:00:00Z"),+ asanaTestTask("t2", "Task Two", "2026-01-02T01:00:00Z"),+ }+ attachments := []asanaAttachment{{+ GID: "a1",+ Name: "plan.pdf",+ DownloadURL: "https://example.com/plan.pdf",+ Size: 10,+ }}+ connector := asanaTestConnector(t, func(ctx context.Context, apiPath string, query url.Values, out any) error {+ switch apiPath {+ case "projects":+ asanaSetEnvelope(out, projects, "")+ case "tasks":+ if query.Get("offset") == "" {+ asanaSetEnvelope(out, []asanaTask{tasks[0]}, "page_2")+ } else {+ asanaSetEnvelope(out, tasks[1:], "")+ }+ case "attachments":+ if query.Get("parent") == "t1" {+ asanaSetEnvelope(out, attachments, "")+ } else {+ asanaSetEnvelope(out, []asanaAttachment{}, "")+ }+ default:+ return fmt.Errorf("unexpected path %q", apiPath)+ }+ return nil+ })+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("first prune NextBatch failed: %v", err)+ }+ if len(first.Documents) != 2 || first.Documents[0].SourceID != "asana:t1" || first.Documents[1].SourceID != "asana:t1:a1" {+ t.Fatalf("first prune docs = %#v", first.Documents)+ }+ second, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("second prune NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "asana:t2" {+ t.Fatalf("second prune docs = %#v", second.Documents)+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("prune final NextBatch = %v, want io.EOF", err)+ }+}++func TestAsanaOpenPruneAdvancesAcrossEmptyTaskPages(t *testing.T) {+ projects := []asanaProject{+ asanaTestProject("p1", "Project One", "team_1"),+ asanaTestProject("p2", "Project Two", "team_1"),+ }+ connector := asanaTestConnector(t, func(ctx context.Context, apiPath string, query url.Values, out any) error {+ switch apiPath {+ case "projects":+ asanaSetEnvelope(out, projects, "")+ case "tasks":+ if query.Get("project") == "p1" {+ if query.Get("offset") == "" {+ asanaSetEnvelope(out, []asanaTask{}, "empty_page")+ } else {+ asanaSetEnvelope(out, []asanaTask{}, "")+ }+ } else {+ asanaSetEnvelope(out, []asanaTask{asanaTestTask("t2", "Task Two", "2026-01-02T00:00:00Z")}, "")+ }+ case "attachments":+ asanaSetEnvelope(out, []asanaAttachment{}, "")+ default:+ if strings.HasPrefix(apiPath, "tasks/") && strings.HasSuffix(apiPath, "/stories") {+ asanaSetEnvelope(out, []asanaStory{}, "")+ return nil+ }+ return fmt.Errorf("unexpected path %q", apiPath)+ }+ return nil+ })+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 || batch.Documents[0].SourceID != "asana:t2" {+ t.Fatalf("documents = %#v, want t2 only", batch.Documents)+ }+ if _, err := session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("final NextBatch = %v, want io.EOF", err)+ }+}++func TestAsanaConnectorRegisteredBuiltIn(t *testing.T) {+ registry := NewRegistry()+ RegisterBuiltIns(registry)+ connector, err := registry.OpenFromConfig("asana", map[string]any{+ "asana_workspace_id": "workspace_1",+ "credentials": map[string]any{+ "asana_api_token_secret": "token",+ },+ })+ if err != nil {+ t.Fatalf("OpenFromConfig failed: %v", err)+ }+ if _, ok := connector.(*AsanaConnector); !ok {+ t.Fatalf("connector type = %T, want *AsanaConnector", connector)+ }+}++func TestAsanaFetchReferenceRejectsOversizedAttachment(t *testing.T) {+ connector, err := NewAsanaConnector(map[string]any{+ "asana_workspace_id": "workspace_1",+ "size_threshold": 10,+ "credentials": map[string]any{+ "asana_api_token_secret": "token",+ },+ })+ if err != nil {+ t.Fatalf("NewAsanaConnector failed: %v", err)+ }+ connector.download = func(ctx context.Context, rawURL string, maxSize int64) ([]byte, error) {+ t.Fatalf("download should not be called for oversized attachment")+ return nil, nil+ }+ refKey, _ := json.Marshal(asanaFetchReference{+ TaskGID: "t1",+ AttachmentGID: "a1",+ Filename: "large.bin",+ DownloadURL: "https://example.com/large.bin",+ Size: 11,+ })+ if _, err := connector.Fetch(context.Background(), FetchReference{Key: string(refKey)}); err == nil {+ t.Fatalf("Fetch unexpectedly succeeded")+ }+}diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 1aa32c2d3b..b958e04fed 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -33,6 +33,7 @@ func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "azure_devops", NewAzureDevOpsConnector)registerBuiltIn(registry, "dropbox", NewDropboxConnector)registerBuiltIn(registry, "box", NewBoxConnector)+ registerBuiltIn(registry, "asana", NewAsanaConnector)registerBuiltIn(registry, "github", NewGitHubConnector)registerBuiltIn(registry, "gitlab", NewGitlabConnector)registerBuiltIn(registry, "gmail", NewGmailConnector)