RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(syncer): add Bitbucket connector with checkpoint resume (#18487)
mkaaadd648d19137c7
diff --git a/internal/syncer/connector/bitbucket.go b/internal/syncer/connector/bitbucket.gonew file mode 100644index 0000000000..c5bcdd2505--- /dev/null+++ b/internal/syncer/connector/bitbucket.go@@ -0,0 +1,1053 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "regexp"+ "sort"+ "strconv"+ "strings"+ "time"++ "ragflow/internal/utility"+)++const (+ defaultBitbucketBatchSize = 32+ bitbucketRequestTimeout = 60 * time.Second+ bitbucketPRPageSize = 50+ bitbucketRepoPageSize = 100+ bitbucketBaseURL = "https://api.bitbucket.org/2.0"+)++var (+ bitbucketRetryTries = 6+ bitbucketRetryBaseDelay = 1 * time.Second+ bitbucketRetryBackoff = 2.0+ bitbucketRetryMaxDelay = 30 * time.Second+ bitbucketSpaceRE = regexp.MustCompile(`\s+`)+)++const bitbucketPRFields = "next,page,pagelen,values.author,values.close_source_branch,values.closed_by,values.comment_count,values.created_on,values.description,values.destination,values.draft,values.id,values.links,values.merge_commit,values.participants,values.reason,values.rendered,values.reviewers,values.source,values.state,values.summary,values.task_count,values.title,values.type,values.updated_on"++const bitbucketSlimPRFields = "next,page,pagelen,values.id"++const bitbucketRepoFields = "next,page,pagelen,values.slug,values.full_name,values.project.key"++// BitbucketConnector reads Bitbucket Cloud pull requests.+type BitbucketConnector struct {+ workspace string+ repositorySlugs []string+ projects []string+ email string+ apiToken string+ batchSize int+ baseURL string+ doJSON func(ctx context.Context, apiURL string, out any) (http.Header, error)+}++// NewBitbucketConnector creates a Bitbucket connector from config.+func NewBitbucketConnector(config map[string]any) (*BitbucketConnector, error) {+ credentials, _ := config["credentials"].(map[string]any)+ return &BitbucketConnector{+ workspace: strings.TrimSpace(stringConfig(config["workspace"])),+ repositorySlugs: splitBitbucketList(stringConfig(config["repository_slugs"])),+ projects: splitBitbucketList(stringConfig(config["projects"])),+ email: strings.TrimSpace(stringConfig(credentials["bitbucket_account_email"])),+ apiToken: strings.TrimSpace(stringConfig(credentials["bitbucket_api_token"])),+ batchSize: configInt(config["batch_size"], defaultBitbucketBatchSize),+ baseURL: bitbucketBaseURL,+ }, nil+}++// Validate validates Bitbucket connector settings and credentials.+func (c *BitbucketConnector) Validate(ctx context.Context) error {+ if c == nil {+ return fmt.Errorf("bitbucket connector is nil")+ }+ if c.workspace == "" {+ return fmt.Errorf("Invalid connector settings: 'workspace' must be provided")+ }+ if c.email == "" {+ return fmt.Errorf("Missing bitbucket_account_email in credentials")+ }+ if c.apiToken == "" {+ return fmt.Errorf("Missing bitbucket_api_token in credentials")+ }+ if c.batchSize <= 0 {+ return fmt.Errorf("batch_size must be a positive integer")+ }+ repos, err := c.listRepos(ctx)+ if err != nil {+ return err+ }+ if len(repos) == 0 {+ return fmt.Errorf("found no repositories for Bitbucket workspace %s", c.workspace)+ }+ return nil+}++// ValidateConnectorSetting validates Bitbucket settings from an unsaved config.+func (c *BitbucketConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ if c == nil {+ return fmt.Errorf("bitbucket connector is nil")+ }+ if c.workspace == "" {+ return fmt.Errorf("Invalid connector settings: 'workspace' must be provided")+ }+ if c.email == "" {+ return fmt.Errorf("Missing bitbucket_account_email in credentials")+ }+ if c.apiToken == "" {+ return fmt.Errorf("Missing bitbucket_api_token in credentials")+ }+ var page map[string]any+ _, err := c.getJSON(ctx, c.apiURL("/repositories/"+url.PathEscape(c.workspace), url.Values{+ "pagelen": {"1"},+ "fields": {"pagelen"},+ }), &page)+ if err != nil {+ var httpErr *bitbucketHTTPError+ if errors.As(err, &httpErr) {+ switch httpErr.Status {+ case http.StatusUnauthorized:+ return fmt.Errorf("Invalid or expired Bitbucket credentials (HTTP 401).")+ case http.StatusForbidden:+ return fmt.Errorf("Insufficient permissions to access Bitbucket workspace (HTTP 403).")+ }+ return err+ }+ return err+ }+ return nil+}++// OpenSync opens one Bitbucket sync session.+func (c *BitbucketConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ repos, err := c.listRepos(ctx)+ if err != nil {+ return nil, err+ }+ if len(repos) == 0 {+ return nil, fmt.Errorf("found no repositories for Bitbucket workspace %s", c.workspace)+ }+ session := &bitbucketSyncSession{+ connector: c,+ repos: repos,+ batchSize: c.batchSize,+ windowStart: request.WindowStart,+ windowEnd: request.WindowEnd,+ }+ session.applyResume(request.Resume)+ return session, nil+}++// OpenPrune opens one complete Bitbucket prune snapshot session.+func (c *BitbucketConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ repos, err := c.listRepos(ctx)+ if err != nil {+ return nil, err+ }+ if len(repos) == 0 {+ return nil, fmt.Errorf("found no repositories for Bitbucket workspace %s", c.workspace)+ }+ return &bitbucketPruneSession{+ connector: c,+ repos: repos,+ batchSize: c.batchSize,+ }, nil+}++// listRepos returns deterministically ordered target repository slugs.+func (c *BitbucketConnector) listRepos(ctx context.Context) ([]string, error) {+ var repos []string+ if len(c.repositorySlugs) > 0 {+ repos = append(repos, c.repositorySlugs...)+ } else if len(c.projects) > 0 {+ for _, project := range c.projects {+ projectRepos, err := c.listWorkspaceRepos(ctx, project)+ if err != nil {+ return nil, err+ }+ repos = append(repos, projectRepos...)+ }+ } else {+ var err error+ repos, err = c.listWorkspaceRepos(ctx, "")+ if err != nil {+ return nil, err+ }+ }+ return uniqueSortedStrings(repos), nil+}++// listWorkspaceRepos lists workspace repositories, optionally filtered by project key.+func (c *BitbucketConnector) listWorkspaceRepos(ctx context.Context, project string) ([]string, error) {+ query := bitbucketRepoListQuery()+ if project != "" {+ query.Set("q", fmt.Sprintf("project.key=%q", project))+ }+ var repos []string+ for pageURL := ""; ; {+ if pageURL == "" {+ pageURL = c.apiURL("/repositories/"+url.PathEscape(c.workspace), query)+ }+ var page bitbucketRepositoryPage+ if _, err := c.getJSON(ctx, pageURL, &page); err != nil {+ return nil, err+ }+ for _, repo := range page.Values {+ if repo.Slug != "" {+ repos = append(repos, repo.Slug)+ }+ }+ if page.Next == "" || len(page.Values) == 0 {+ break+ }+ if err := c.validateBitbucketHost(page.Next); err != nil {+ return nil, err+ }+ pageURL = page.Next+ }+ return repos, nil+}++// listPullRequestPage returns one Bitbucket pull request page.+func (c *BitbucketConnector) listPullRequestPage(ctx context.Context, repo, pageURL string, windowStart *time.Time, windowEnd time.Time) (bitbucketPullRequestPage, error) {+ if pageURL == "" {+ pageURL = c.apiURL("/repositories/"+url.PathEscape(c.workspace)+"/"+url.PathEscape(repo)+"/pullrequests", bitbucketPRListQuery(windowStart, windowEnd))+ }+ if err := c.validateBitbucketHost(pageURL); err != nil {+ return bitbucketPullRequestPage{}, err+ }+ var page bitbucketPullRequestPage+ if _, err := c.getJSON(ctx, pageURL, &page); err != nil {+ return bitbucketPullRequestPage{}, err+ }+ return page, nil+}++// listSlimPullRequestPage returns one Bitbucket pull request page for prune snapshots.+func (c *BitbucketConnector) listSlimPullRequestPage(ctx context.Context, repo, pageURL string) (bitbucketPullRequestPage, error) {+ if pageURL == "" {+ query := url.Values{+ "fields": {bitbucketSlimPRFields},+ "pagelen": {strconv.Itoa(bitbucketPRPageSize)},+ "sort": {"updated_on"},+ "q": {bitbucketPullRequestStateQuery()},+ }+ pageURL = c.apiURL("/repositories/"+url.PathEscape(c.workspace)+"/"+url.PathEscape(repo)+"/pullrequests", query)+ }+ if err := c.validateBitbucketHost(pageURL); err != nil {+ return bitbucketPullRequestPage{}, err+ }+ var page bitbucketPullRequestPage+ if _, err := c.getJSON(ctx, pageURL, &page); err != nil {+ return bitbucketPullRequestPage{}, err+ }+ return page, nil+}++// validateBitbucketHost rejects a server-supplied pagination URL whose host does+// not match the configured Bitbucket host, so authenticated requests cannot be+// redirected at an attacker-controlled host.+func (c *BitbucketConnector) validateBitbucketHost(rawURL string) error {+ parsed, err := url.Parse(strings.TrimSpace(rawURL))+ if err != nil {+ return fmt.Errorf("bitbucket: invalid pagination URL: %w", err)+ }+ base, err := url.Parse(strings.TrimSpace(c.baseURL))+ if err != nil {+ return fmt.Errorf("bitbucket: invalid configured base URL: %w", err)+ }+ if !strings.EqualFold(parsed.Scheme, base.Scheme) {+ return fmt.Errorf("bitbucket: pagination URL scheme %q does not match configured scheme %q", parsed.Scheme, base.Scheme)+ }+ if !strings.EqualFold(parsed.Hostname(), base.Hostname()) {+ return fmt.Errorf("bitbucket: pagination URL host %q does not match configured host %q", parsed.Hostname(), base.Hostname())+ }+ if effectiveBitbucketPort(parsed) != effectiveBitbucketPort(base) {+ return fmt.Errorf("bitbucket: pagination URL port %q does not match configured port %q", effectiveBitbucketPort(parsed), effectiveBitbucketPort(base))+ }+ return nil+}++// effectiveBitbucketPort returns the explicit port, falling back to the+// scheme's default port so an explicit default port still matches.+func effectiveBitbucketPort(u *url.URL) string {+ if p := u.Port(); p != "" {+ return p+ }+ switch strings.ToLower(u.Scheme) {+ case "http":+ return "80"+ case "https":+ return "443"+ }+ return ""+}++// apiURL builds a Bitbucket API URL.+func (c *BitbucketConnector) apiURL(path string, query url.Values) string {+ path = strings.TrimLeft(path, "/")+ u := strings.TrimRight(c.baseURL, "/") + "/" + path+ if len(query) == 0 {+ return u+ }+ return u + "?" + query.Encode()+}++// getJSON fetches a Bitbucket API JSON response into out.+func (c *BitbucketConnector) getJSON(ctx context.Context, apiURL string, out any) (http.Header, error) {+ if c.doJSON != nil {+ return c.doJSON(ctx, apiURL, out)+ }+ return c.getJSONWithRetry(ctx, apiURL, out)+}++func (c *BitbucketConnector) getJSONWithRetry(ctx context.Context, apiURL string, out any) (http.Header, error) {+ delay := bitbucketRetryBaseDelay+ var lastErr error+ for attempt := 0; attempt < bitbucketRetryTries; attempt++ {+ resp, err := c.get(ctx, apiURL)+ if err != nil {+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }+ lastErr = err+ if attempt == bitbucketRetryTries-1 {+ break+ }+ if err := sleepFor(ctx, delay); err != nil {+ return nil, err+ }+ delay = bitbucketNextDelay(delay)+ continue+ }++ if resp.StatusCode == http.StatusTooManyRequests {+ retryAfter := bitbucketRetryAfter(resp)+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))+ resp.Body.Close()+ lastErr = &bitbucketHTTPError{Status: resp.StatusCode}+ if attempt == bitbucketRetryTries-1 {+ break+ }+ wait := retryAfter+ if wait <= 0 {+ wait = delay+ }+ if err := sleepFor(ctx, wait); err != nil {+ return nil, err+ }+ delay = bitbucketNextDelay(delay)+ continue+ }++ if resp.StatusCode >= http.StatusInternalServerError {+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ resp.Body.Close()+ lastErr = &bitbucketHTTPError{Status: resp.StatusCode, Body: string(body)}+ if attempt == bitbucketRetryTries-1 {+ break+ }+ if err := sleepFor(ctx, delay); err != nil {+ return nil, err+ }+ delay = bitbucketNextDelay(delay)+ continue+ }++ if resp.StatusCode >= http.StatusBadRequest {+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ resp.Body.Close()+ return nil, &bitbucketHTTPError{Status: resp.StatusCode, Body: string(body)}+ }++ headers := resp.Header.Clone()+ if err := json.NewDecoder(resp.Body).Decode(out); err != nil {+ resp.Body.Close()+ return nil, err+ }+ resp.Body.Close()+ return headers, nil+ }+ return nil, lastErr+}++// get performs one SSRF-safe authenticated GET request.+func (c *BitbucketConnector) get(ctx context.Context, apiURL string) (*http.Response, error) {+ hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)+ if err != nil {+ return nil, err+ }+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)+ if err != nil {+ return nil, err+ }+ req.Header.Set("Accept", "application/json")+ req.SetBasicAuth(c.email, c.apiToken)+ client := utility.PinnedHTTPClient(hostname, resolvedIP, bitbucketRequestTimeout)+ return client.Do(req)+}++func sleepFor(ctx context.Context, duration time.Duration) error {+ if duration <= 0 {+ return nil+ }+ select {+ case <-ctx.Done():+ return ctx.Err()+ case <-time.After(duration):+ return nil+ }+}++func bitbucketNextDelay(delay time.Duration) time.Duration {+ next := time.Duration(float64(delay) * bitbucketRetryBackoff)+ if next > bitbucketRetryMaxDelay {+ return bitbucketRetryMaxDelay+ }+ return next+}++func bitbucketRetryAfter(resp *http.Response) time.Duration {+ raw := strings.TrimSpace(resp.Header.Get("Retry-After"))+ if raw == "" {+ return 0+ }+ seconds, err := strconv.ParseFloat(raw, 64)+ if err != nil || seconds < 0 {+ return 0+ }+ wait := time.Duration(seconds * float64(time.Second))+ if wait > bitbucketRetryMaxDelay {+ return bitbucketRetryMaxDelay+ }+ return wait+}++func bitbucketPullRequestStateQuery() string {+ return `(state = "OPEN" OR state = "MERGED" OR state = "DECLINED")`+}++func bitbucketPRListQuery(windowStart *time.Time, windowEnd time.Time) url.Values {+ query := url.Values{+ "fields": {bitbucketPRFields},+ "pagelen": {strconv.Itoa(bitbucketPRPageSize)},+ "sort": {"updated_on"},+ }+ stateQuery := bitbucketPullRequestStateQuery()+ if windowStart != nil && !windowEnd.IsZero() {+ query.Set("q", fmt.Sprintf("%s AND (updated_on > %q AND updated_on <= %q)", stateQuery, bitbucketISO(*windowStart), bitbucketISO(windowEnd)))+ } else {+ query.Set("q", stateQuery)+ }+ return query+}++func bitbucketRepoListQuery() url.Values {+ return url.Values{+ "fields": {bitbucketRepoFields},+ "pagelen": {strconv.Itoa(bitbucketRepoPageSize)},+ "sort": {"full_name"},+ }+}++func bitbucketISO(t time.Time) string {+ return t.UTC().Format(time.RFC3339Nano)+}++type bitbucketHTTPError struct {+ Status int+ Body string+}++func (e *bitbucketHTTPError) Error() string {+ if strings.TrimSpace(e.Body) != "" {+ return fmt.Sprintf("Bitbucket API returned HTTP %d: %s", e.Status, strings.TrimSpace(e.Body))+ }+ return fmt.Sprintf("Bitbucket API returned HTTP %d", e.Status)+}++type bitbucketSyncSession struct {+ connector *BitbucketConnector+ repos []string+ repoIndex int+ pageURL string+ batchSize int+ windowStart *time.Time+ windowEnd time.Time+ buffer []bitbucketBufferedDocument+ resumeRepo string+ resumePageURL string+ resumeOffset int+ resumeSource string+}++// NextBatch returns the next Bitbucket document batch.+func (s *bitbucketSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ documents := make([]SourceDocument, 0, s.batchSize)+ var checkpoint *SyncCheckpoint+ if len(s.buffer) > 0 {+ n := s.batchSize+ if n > len(s.buffer) {+ n = len(s.buffer)+ }+ for _, buffered := range s.buffer[:n] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }+ s.buffer = s.buffer[n:]+ }++ for len(documents) < s.batchSize {+ if s.repoIndex >= len(s.repos) {+ if len(documents) == 0 {+ return SyncBatch{}, io.EOF+ }+ break+ }+ batch, err := s.nextDocumentPage(ctx)+ if err != nil {+ return SyncBatch{}, err+ }+ if len(batch) == 0 {+ if s.repoIndex >= len(s.repos) && len(documents) == 0 {+ return SyncBatch{}, io.EOF+ }+ continue+ }+ remaining := s.batchSize - len(documents)+ if len(batch) > remaining {+ for _, buffered := range batch[:remaining] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }+ s.buffer = append(s.buffer, batch[remaining:]...)+ break+ }+ for _, buffered := range batch {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }+ }+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil+}++// Close closes the Bitbucket sync session.+func (s *bitbucketSyncSession) Close() error {+ return nil+}++func (s *bitbucketSyncSession) nextDocumentPage(ctx context.Context) ([]bitbucketBufferedDocument, error) {+ if s.repoIndex >= len(s.repos) {+ return nil, nil+ }+ repo := s.repos[s.repoIndex]+ pageURL := s.pageURL+ page, err := s.connector.listPullRequestPage(ctx, repo, pageURL, s.windowStart, s.windowEnd)+ if err != nil {+ return nil, err+ }+ documents := make([]bitbucketBufferedDocument, 0, len(page.Values))+ pageOffset := 0+ for _, pr := range page.Values {+ updatedAt := parseBitbucketTime(pr.UpdatedOn)+ if beforeOrAtWindowStart(updatedAt, s.windowStart) {+ continue+ }+ if afterWindowEnd(updatedAt, s.windowEnd) {+ continue+ }+ doc := pr.toSourceDocument(s.connector.workspace, repo)+ pageOffset+++ documents = append(documents, bitbucketBufferedDocument{+ document: doc,+ checkpoint: bitbucketSyncCheckpoint(repo, pageURL, pageOffset, doc),+ offset: pageOffset,+ sourceID: doc.SourceID,+ })+ }+ documents = s.filterResumedDocuments(repo, pageURL, documents)+ if page.Next == "" || len(page.Values) == 0 {+ s.advanceRepo()+ } else {+ s.pageURL = page.Next+ }+ return documents, nil+}++// advanceRepo moves a Bitbucket sync session to the next repository.+func (s *bitbucketSyncSession) advanceRepo() {+ s.repoIndex+++ s.pageURL = ""+ s.clearResume()+}++// applyResume advances a sync session to the last committed Bitbucket position.+func (s *bitbucketSyncSession) applyResume(checkpoint *SyncCheckpoint) {+ if checkpoint == nil || checkpoint.Cursor == "" {+ return+ }+ var cursor bitbucketSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ return+ }+ if cursor.RepoSlug == "" {+ return+ }+ for index, repo := range s.repos {+ if repo != cursor.RepoSlug {+ continue+ }+ s.repoIndex = index+ s.pageURL = cursor.PageURL+ s.resumeRepo = repo+ s.resumePageURL = cursor.PageURL+ s.resumeOffset = cursor.PageOffset+ s.resumeSource = firstNonEmpty(cursor.SourceID, checkpoint.SourceID)+ return+ }+}++// filterResumedDocuments drops documents through the committed checkpoint.+func (s *bitbucketSyncSession) filterResumedDocuments(repo, pageURL string, candidates []bitbucketBufferedDocument) []bitbucketBufferedDocument {+ if s.resumeRepo == "" || repo != s.resumeRepo || pageURL != s.resumePageURL {+ return candidates+ }+ if s.resumeSource != "" {+ for index, candidate := range candidates {+ if candidate.sourceID == s.resumeSource {+ s.clearResume()+ return candidates[index+1:]+ }+ }+ }+ if s.resumeOffset > 0 {+ filtered := candidates[:0]+ for _, candidate := range candidates {+ if candidate.offset > s.resumeOffset {+ filtered = append(filtered, candidate)+ }+ }+ s.clearResume()+ return filtered+ }+ s.clearResume()+ return candidates+}++func (s *bitbucketSyncSession) clearResume() {+ s.resumeRepo = ""+ s.resumePageURL = ""+ s.resumeOffset = 0+ s.resumeSource = ""+}++type bitbucketPruneSession struct {+ connector *BitbucketConnector+ repos []string+ repoIndex int+ pageURL string+ batchSize int+ buffer []SlimDocument+}++// NextBatch returns the next Bitbucket prune snapshot batch.+func (s *bitbucketPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ documents := make([]SlimDocument, 0, s.batchSize)+ if len(s.buffer) > 0 {+ n := s.batchSize+ if n > len(s.buffer) {+ n = len(s.buffer)+ }+ documents = append(documents, s.buffer[:n]...)+ s.buffer = s.buffer[n:]+ }+ for len(documents) < s.batchSize {+ if s.repoIndex >= len(s.repos) {+ if len(documents) == 0 {+ return PruneBatch{}, io.EOF+ }+ break+ }+ batch, err := s.nextSlimPage(ctx)+ if err != nil {+ return PruneBatch{}, err+ }+ if len(batch) == 0 {+ if s.repoIndex >= len(s.repos) && len(documents) == 0 {+ return PruneBatch{}, io.EOF+ }+ continue+ }+ remaining := s.batchSize - len(documents)+ if len(batch) > remaining {+ documents = append(documents, batch[:remaining]...)+ s.buffer = append(s.buffer, batch[remaining:]...)+ break+ }+ documents = append(documents, batch...)+ }+ return PruneBatch{Documents: documents}, nil+}++// Close closes the Bitbucket prune session.+func (s *bitbucketPruneSession) Close() error {+ return nil+}++func (s *bitbucketPruneSession) nextSlimPage(ctx context.Context) ([]SlimDocument, error) {+ if s.repoIndex >= len(s.repos) {+ return nil, nil+ }+ repo := s.repos[s.repoIndex]+ pageURL := s.pageURL+ page, err := s.connector.listSlimPullRequestPage(ctx, repo, pageURL)+ if err != nil {+ return nil, err+ }+ documents := make([]SlimDocument, 0, len(page.Values))+ for _, pr := range page.Values {+ documents = append(documents, SlimDocument{SourceID: bitbucketSourceID(s.connector.workspace, repo, pr.ID)})+ }+ if page.Next == "" || len(page.Values) == 0 {+ s.repoIndex+++ s.pageURL = ""+ } else {+ s.pageURL = page.Next+ }+ return documents, nil+}++type bitbucketRepositoryPage struct {+ Values []bitbucketRepository `json:"values"`+ Next string `json:"next"`+}++type bitbucketRepository struct {+ Slug string `json:"slug"`+}++type bitbucketPullRequestPage struct {+ Values []bitbucketPullRequest `json:"values"`+ Next string `json:"next"`+}++type bitbucketPullRequest struct {+ ID int `json:"id"`+ Title string `json:"title"`+ Description string `json:"description"`+ State string `json:"state"`+ Reason string `json:"reason"`+ Draft bool `json:"draft"`+ Author *bitbucketUser `json:"author"`+ Reviewers []bitbucketUser `json:"reviewers"`+ Participants []bitbucketParticipant `json:"participants"`+ CommentCount int `json:"comment_count"`+ TaskCount int `json:"task_count"`+ CreatedOn string `json:"created_on"`+ UpdatedOn string `json:"updated_on"`+ Source bitbucketBranchRef `json:"source"`+ Destination bitbucketBranchRef `json:"destination"`+ Links bitbucketPRLinks `json:"links"`+ ClosedBy *bitbucketUser `json:"closed_by"`+ CloseSourceBranch bool `json:"close_source_branch"`+}++type bitbucketUser struct {+ DisplayName string `json:"display_name"`+ Nickname string `json:"nickname"`+}++type bitbucketParticipant struct {+ User bitbucketUser `json:"user"`+ Approved bool `json:"approved"`+}++type bitbucketBranchRef struct {+ Branch struct {+ Name string `json:"name"`+ } `json:"branch"`+}++type bitbucketPRLinks struct {+ HTML struct {+ Href string `json:"href"`+ } `json:"html"`+}++type bitbucketSyncCursor struct {+ RepoSlug string `json:"repo_slug"`+ PageURL string `json:"page_url,omitempty"`+ PageOffset int `json:"page_offset,omitempty"`+ SourceID string `json:"source_id,omitempty"`+}++type bitbucketBufferedDocument struct {+ document SourceDocument+ checkpoint *SyncCheckpoint+ offset int+ sourceID string+}++func bitbucketSyncCheckpoint(repo, pageURL string, offset int, doc SourceDocument) *SyncCheckpoint {+ cursor, err := json.Marshal(bitbucketSyncCursor{+ RepoSlug: repo,+ PageURL: pageURL,+ PageOffset: offset,+ SourceID: doc.SourceID,+ })+ if err != nil {+ return nil+ }+ updatedAt := doc.UpdatedAt+ return &SyncCheckpoint{+ Cursor: string(cursor),+ SourceID: doc.SourceID,+ UpdatedAt: &updatedAt,+ }+}++func (p bitbucketPullRequest) toSourceDocument(workspace, repo string) SourceDocument {+ id := p.ID+ title := strings.TrimSpace(p.Title)+ if title == "" {+ title = fmt.Sprintf("PR %d", id)+ }+ body := bitbucketPRBody(p, title)+ updatedAt := parseBitbucketTime(p.UpdatedOn)+ reviewers := bitbucketUserNames(p.Reviewers)+ approvedBy := bitbucketApprovedBy(p.Participants)+ author := bitbucketUserName(p.Author)+ closedBy := bitbucketUserName(p.ClosedBy)+ sourceBranch := p.Source.Branch.Name+ destinationBranch := p.Destination.Branch.Name+ link := p.Links.HTML.Href+ if link == "" {+ link = fmt.Sprintf("https://bitbucket.org/%s/%s/pull-requests/%d", workspace, repo, id)+ }+ return SourceDocument{+ SourceID: bitbucketSourceID(workspace, repo, id),+ SemanticIdentifier: bitbucketSemanticIdentifier(id, title),+ Extension: ".md",+ Blob: []byte(body),+ UpdatedAt: updatedAt,+ SizeBytes: int64(len(body)),+ Metadata: map[string]any{+ "object_type": "PullRequest",+ "workspace": workspace,+ "repository": repo,+ "pr_key": fmt.Sprintf("%s/%s#%d", workspace, repo, id),+ "id": strconv.Itoa(id),+ "title": title,+ "state": p.State,+ "draft": strconv.FormatBool(p.Draft),+ "link": link,+ "author": author,+ "reviewers": reviewers,+ "approved_by": approvedBy,+ "comment_count": strconv.Itoa(p.CommentCount),+ "task_count": strconv.Itoa(p.TaskCount),+ "created_on": p.CreatedOn,+ "updated_on": p.UpdatedOn,+ "source_branch": sourceBranch,+ "destination_branch": destinationBranch,+ "closed_by": closedBy,+ "close_source_branch": strconv.FormatBool(p.CloseSourceBranch),+ },+ Fingerprint: stableFingerprint(map[string]any{+ "object_type": "PullRequest",+ "workspace": workspace,+ "repository": repo,+ "pr_id": id,+ "title": title,+ "description": p.Description,+ "state": p.State,+ "reason": p.Reason,+ "draft": p.Draft,+ "updated_on": p.UpdatedOn,+ "created_on": p.CreatedOn,+ "source_branch": sourceBranch,+ "destination_branch": destinationBranch,+ "author": author,+ "reviewers": reviewers,+ "approved_by": approvedBy,+ "comment_count": p.CommentCount,+ "task_count": p.TaskCount,+ "closed_by": closedBy,+ "close_source_branch": p.CloseSourceBranch,+ }),+ }+}++func bitbucketSourceID(workspace, repo string, id int) string {+ return fmt.Sprintf("bitbucket:%s:%s:pr:%d", workspace, repo, id)+}++func bitbucketSemanticIdentifier(id int, title string) string {+ return fmt.Sprintf("#%d: %s", id, sanitizeBitbucketName(title))+}++func bitbucketPRBody(p bitbucketPullRequest, title string) string {+ var body strings.Builder+ fmt.Fprintf(&body, "Pull Request Information:\n- Pull Request ID: %d\n- Title: %s\n- State: %s", p.ID, title, p.State)+ if p.Draft {+ body.WriteString(" (Draft)")+ }+ body.WriteString("\n")+ if strings.EqualFold(p.State, "DECLINED") {+ reason := strings.TrimSpace(p.Reason)+ if reason == "" {+ reason = "N/A"+ }+ fmt.Fprintf(&body, "- Reason: %s\n", reason)+ }+ fmt.Fprintf(&body, "- Author: %s\n", bitbucketBodyAuthor(p.Author))+ reviewers := bitbucketUserNames(p.Reviewers)+ if len(reviewers) == 0 {+ body.WriteString("- Reviewers: N/A\n")+ } else {+ fmt.Fprintf(&body, "- Reviewers: %s\n", strings.Join(reviewers, ", "))+ }+ fmt.Fprintf(&body, "- Branch: %s -> %s\n", p.Source.Branch.Name, p.Destination.Branch.Name)+ fmt.Fprintf(&body, "- Created: %s\n", bitbucketDateOrNA(p.CreatedOn))+ fmt.Fprintf(&body, "- Updated: %s", bitbucketDateOrNA(p.UpdatedOn))+ if description := strings.TrimSpace(p.Description); description != "" {+ body.WriteString("\n\nDescription:\n")+ body.WriteString(description)+ }+ return body.String()+}++func bitbucketBodyAuthor(user *bitbucketUser) string {+ if user == nil {+ return "N/A"+ }+ return bitbucketUserName(user)+}++func bitbucketUserName(user *bitbucketUser) string {+ if user == nil {+ return ""+ }+ if user.DisplayName != "" {+ return user.DisplayName+ }+ if user.Nickname != "" {+ return user.Nickname+ }+ return "unknown"+}++func bitbucketUserNames(users []bitbucketUser) []string {+ out := make([]string, 0, len(users))+ for i := range users {+ out = append(out, bitbucketUserName(&users[i]))+ }+ sort.Strings(out)+ return out+}++func bitbucketApprovedBy(participants []bitbucketParticipant) []string {+ out := make([]string, 0, len(participants))+ for _, participant := range participants {+ if participant.Approved {+ out = append(out, bitbucketUserName(&participant.User))+ }+ }+ sort.Strings(out)+ return out+}++func bitbucketDateOrNA(value string) string {+ if value == "" {+ return "N/A"+ }+ return strings.Split(value, "T")[0]+}++func parseBitbucketTime(value string) time.Time {+ value = strings.TrimSpace(value)+ if value == "" {+ return time.Time{}+ }+ for _, layout := range []string{time.RFC3339Nano, "2006-01-02 15:04:05", "2006-01-02T15:04:05"} {+ if parsed, err := time.Parse(layout, value); err == nil {+ return parsed.UTC()+ }+ }+ return time.Time{}+}++// sanitizeBitbucketName mirrors the Bitbucket PR filename cleanup semantics.+func sanitizeBitbucketName(name string) string {+ name = strings.TrimSpace(name)+ if name == "" {+ name = "PR"+ }+ replacer := strings.NewReplacer("\\", " ", "?", " ", "#", " ", "%", " ", "*", " ", ":", " ", "|", " ", "<", " ", ">", " ", `"`, " ")+ name = replacer.Replace(name)+ name = strings.ReplaceAll(name, "/", " ")+ name = bitbucketSpaceRE.ReplaceAllString(name, " ")+ name = strings.TrimSpace(name)+ const maxNameRunes = 200+ if runes := []rune(name); len(runes) > maxNameRunes {+ name = strings.TrimSpace(string(runes[:maxNameRunes]))+ }+ if name == "" {+ name = "PR"+ }+ return name + ".md"+}++func splitBitbucketList(value string) []string {+ parts := strings.Split(value, ",")+ out := make([]string, 0, len(parts))+ for _, part := range parts {+ part = strings.TrimSpace(part)+ if part != "" {+ out = append(out, part)+ }+ }+ return out+}++func uniqueSortedStrings(values []string) []string {+ if len(values) == 0 {+ return []string{}+ }+ values = append([]string(nil), values...)+ sort.Strings(values)+ unique := values[:1]+ for _, value := range values[1:] {+ if value != unique[len(unique)-1] {+ unique = append(unique, value)+ }+ }+ return unique+}diff --git a/internal/syncer/connector/bitbucket_test.go b/internal/syncer/connector/bitbucket_test.gonew file mode 100644index 0000000000..846184c10e--- /dev/null+++ b/internal/syncer/connector/bitbucket_test.go@@ -0,0 +1,718 @@+package connector++import (+ "context"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "net/http/httptest"+ "net/url"+ "strings"+ "sync/atomic"+ "testing"+ "time"++ "ragflow/internal/utility"+)++func TestNewBitbucketConnectorParsesConfig(t *testing.T) {+ connector, err := NewBitbucketConnector(map[string]any{+ "workspace": " acme ",+ "repository_slugs": "repo-a, repo-b,",+ "projects": "P1, P2",+ "batch_size": 7,+ "credentials": map[string]any{+ "bitbucket_account_email": "[email protected]",+ "bitbucket_api_token": "token",+ },+ })+ if err != nil {+ t.Fatalf("NewBitbucketConnector failed: %v", err)+ }+ if connector.workspace != "acme" {+ t.Fatalf("workspace = %q, want acme", connector.workspace)+ }+ if len(connector.repositorySlugs) != 2 || connector.repositorySlugs[0] != "repo-a" || connector.repositorySlugs[1] != "repo-b" {+ t.Fatalf("repositorySlugs = %v", connector.repositorySlugs)+ }+ if len(connector.projects) != 2 || connector.projects[0] != "P1" || connector.projects[1] != "P2" {+ t.Fatalf("projects = %v", connector.projects)+ }+ if connector.email != "[email protected]" || connector.apiToken != "token" {+ t.Fatalf("credentials = %q %q", connector.email, connector.apiToken)+ }+ if connector.batchSize != 7 {+ t.Fatalf("batch_size = %d, want 7", connector.batchSize)+ }+ if connector.baseURL != bitbucketBaseURL {+ t.Fatalf("baseURL = %q, want %q", connector.baseURL, bitbucketBaseURL)+ }+}++func TestBitbucketListReposUsesConfiguredSlugsBeforeProjects(t *testing.T) {+ connector := &BitbucketConnector{+ workspace: "acme",+ repositorySlugs: []string{"repo-b", "repo-a", "repo-a"},+ projects: []string{"P1"},+ baseURL: "https://api.bitbucket.test",+ doJSON: func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ t.Fatalf("repository slug config should not call the workspace API")+ return nil, nil+ },+ }+ repos, err := connector.listRepos(context.Background())+ if err != nil {+ t.Fatalf("listRepos failed: %v", err)+ }+ if len(repos) != 2 || repos[0] != "repo-a" || repos[1] != "repo-b" {+ t.Fatalf("repos = %v, want [repo-a repo-b]", repos)+ }+}++func TestBitbucketListReposFiltersByProjectKey(t *testing.T) {+ connector := &BitbucketConnector{+ workspace: "acme",+ projects: []string{"P1", "P2"},+ baseURL: "https://api.bitbucket.test",+ doJSON: func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ parsed, err := url.Parse(apiURL)+ if err != nil {+ t.Fatalf("parse api url: %v", err)+ }+ if parsed.Path != "/repositories/acme" {+ t.Fatalf("path = %q", parsed.Path)+ }+ project := parsed.Query().Get("q")+ if project != `project.key="P1"` && project != `project.key="P2"` {+ t.Fatalf("project query = %q", project)+ }+ slug := "repo-a"+ if strings.Contains(project, "P2") {+ slug = "repo-b"+ }+ body, _ := json.Marshal(bitbucketRepositoryPage{+ Values: []bitbucketRepository{{Slug: slug}},+ })+ return nil, json.Unmarshal(body, out)+ },+ }+ repos, err := connector.listRepos(context.Background())+ if err != nil {+ t.Fatalf("listRepos failed: %v", err)+ }+ if len(repos) != 2 || repos[0] != "repo-a" || repos[1] != "repo-b" {+ t.Fatalf("repos = %v, want [repo-a repo-b]", repos)+ }+}++func TestBitbucketPaginationRejectsForeignHost(t *testing.T) {+ connector := &BitbucketConnector{+ workspace: "acme",+ baseURL: "https://api.bitbucket.test",+ }++ // listWorkspaceRepos refuses to follow a server-supplied next URL on a+ // different host than the configured Bitbucket API.+ connector.doJSON = func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ body, _ := json.Marshal(bitbucketRepositoryPage{+ Values: []bitbucketRepository{{Slug: "repo-a"}},+ Next: "https://evil.example.com/repositories/acme",+ })+ return nil, json.Unmarshal(body, out)+ }+ if _, err := connector.listWorkspaceRepos(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "api.bitbucket.test") {+ t.Fatalf("listWorkspaceRepos error = %v, want host mismatch", err)+ }++ // A same-host URL on a different scheme is still rejected.+ connector.doJSON = func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ body, _ := json.Marshal(bitbucketRepositoryPage{+ Values: []bitbucketRepository{{Slug: "repo-a"}},+ Next: "http://api.bitbucket.test/repositories/acme",+ })+ return nil, json.Unmarshal(body, out)+ }+ if _, err := connector.listWorkspaceRepos(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "scheme") {+ t.Fatalf("listWorkspaceRepos error = %v, want scheme mismatch", err)+ }++ // A same-host URL on a different port is still rejected.+ connector.doJSON = func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ body, _ := json.Marshal(bitbucketRepositoryPage{+ Values: []bitbucketRepository{{Slug: "repo-a"}},+ Next: "https://api.bitbucket.test:8443/repositories/acme",+ })+ return nil, json.Unmarshal(body, out)+ }+ if _, err := connector.listWorkspaceRepos(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "port") {+ t.Fatalf("listWorkspaceRepos error = %v, want port mismatch", err)+ }++ // The pull-request page helpers stop before requesting a foreign host.+ called := false+ connector.doJSON = func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ called = true+ return http.Header{}, nil+ }+ if _, err := connector.listPullRequestPage(context.Background(), "repo-a", "https://evil.example.com/repositories/acme/repo-a/pullrequests", nil, time.Time{}); err == nil {+ t.Fatalf("listPullRequestPage should reject a foreign host")+ }+ if called {+ t.Fatalf("listPullRequestPage requested a foreign host")+ }++ called = false+ if _, err := connector.listSlimPullRequestPage(context.Background(), "repo-a", "https://evil.example.com/repositories/acme/repo-a/pullrequests"); err == nil {+ t.Fatalf("listSlimPullRequestPage should reject a foreign host")+ }+ if called {+ t.Fatalf("listSlimPullRequestPage requested a foreign host")+ }+}++func TestBitbucketPRListQueryBuildsIncrementalWindow(t *testing.T) {+ start := mustTime(t, "2026-01-02T12:00:00Z")+ end := mustTime(t, "2026-01-04T00:00:00Z")+ query := bitbucketPRListQuery(&start, end)+ q := query.Get("q")+ for _, want := range []string{+ `(state = "OPEN" OR state = "MERGED" OR state = "DECLINED")`,+ "updated_on >",+ "updated_on <=",+ } {+ if !strings.Contains(q, want) {+ t.Fatalf("q = %q, want contains %q", q, want)+ }+ }+ if query.Get("sort") != "updated_on" {+ t.Fatalf("sort = %q", query.Get("sort"))+ }+}++func TestBitbucketPullRequestToSourceDocument(t *testing.T) {+ pr := bitbucketPullRequest{+ ID: 7,+ Title: "Add: Bitbucket PR",+ Description: "Sync PR body",+ State: "MERGED",+ Draft: true,+ Author: &bitbucketUser{DisplayName: "Alice", Nickname: "alice"},+ Reviewers: []bitbucketUser{+ {DisplayName: "Zoe"},+ {DisplayName: "Bob"},+ },+ Participants: []bitbucketParticipant{+ {User: bitbucketUser{DisplayName: "Zoe"}, Approved: true},+ {User: bitbucketUser{DisplayName: "Bill"}, Approved: false},+ },+ CommentCount: 3,+ TaskCount: 2,+ CreatedOn: "2026-01-01T10:00:00+00:00",+ UpdatedOn: "2026-01-03T00:00:00+00:00",+ Source: bitbucketBranchRef{Branch: struct {+ Name string `json:"name"`+ }{Name: "feature"}},+ Destination: bitbucketBranchRef{Branch: struct {+ Name string `json:"name"`+ }{Name: "main"}},+ Links: bitbucketPRLinks{HTML: struct {+ Href string `json:"href"`+ }{Href: "https://bitbucket.org/acme/repo/pull-requests/7"}},+ }+ first := pr.toSourceDocument("acme", "repo")+ second := pr.toSourceDocument("acme", "repo")+ if first.SourceID != "bitbucket:acme:repo:pr:7" {+ t.Fatalf("source id = %q", first.SourceID)+ }+ if first.SemanticIdentifier != "#7: Add Bitbucket PR.md" {+ t.Fatalf("semantic identifier = %q", first.SemanticIdentifier)+ }+ if first.Extension != ".md" {+ t.Fatalf("extension = %q", first.Extension)+ }+ if first.Fingerprint == "" || first.Fingerprint != second.Fingerprint {+ t.Fatalf("fingerprint unstable: %q %q", first.Fingerprint, second.Fingerprint)+ }+ if !strings.Contains(string(first.Blob), "Pull Request Information") ||+ !strings.Contains(string(first.Blob), "Description:\nSync PR body") {+ t.Fatalf("blob missing PR content: %s", first.Blob)+ }+ reviewers, _ := first.Metadata["reviewers"].([]string)+ if len(reviewers) != 2 || reviewers[0] != "Bob" || reviewers[1] != "Zoe" {+ t.Fatalf("reviewers = %v", reviewers)+ }++ changed := pr+ changed.Title = "Add Bitbucket PR v2"+ if got := changed.toSourceDocument("acme", "repo").Fingerprint; got == first.Fingerprint {+ t.Fatalf("fingerprint did not change after title update")+ }+}++func TestBitbucketConnectorOpenSyncUsesWindowAndFingerprint(t *testing.T) {+ connector := mustBitbucketConnector(t, 1, "repo-a", "repo-b")+ connector.doJSON = bitbucketFixtureDoJSON(t)++ start := mustTime(t, "2026-01-02T12:00:00Z")+ end := mustTime(t, "2026-01-04T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ doc := batch.Documents[0]+ if doc.SourceID != "bitbucket:acme:repo-a:pr:1" {+ t.Fatalf("source id = %q", doc.SourceID)+ }+ if doc.Fingerprint == "" {+ t.Fatalf("fingerprint is empty")+ }+ second, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch second failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "bitbucket:acme:repo-b:pr:3" {+ t.Fatalf("second documents = %+v, want PR 3", second.Documents)+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++func TestBitbucketConnectorOpenSyncResumesAfterCheckpoint(t *testing.T) {+ connector := mustBitbucketConnector(t, 1, "repo-a", "repo-b")+ connector.doJSON = bitbucketFixtureDoJSON(t)++ end := mustTime(t, "2026-01-04T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch first failed: %v", err)+ }+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:1" {+ t.Fatalf("first documents = %+v, want PR 1", first.Documents)+ }+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "bitbucket:acme:repo-a:pr:1" {+ t.Fatalf("first checkpoint = %+v, want PR 1", first.Checkpoint)+ }++ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end, Resume: first.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:2" {+ t.Fatalf("resume documents = %+v, want PR 2", second.Documents)+ }+ if second.Checkpoint == nil || second.Checkpoint.SourceID != "bitbucket:acme:repo-a:pr:2" {+ t.Fatalf("resume checkpoint = %+v, want PR 2", second.Checkpoint)+ }+ third, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch third failed: %v", err)+ }+ if len(third.Documents) != 1 || third.Documents[0].SourceID != "bitbucket:acme:repo-b:pr:3" {+ t.Fatalf("resume documents = %+v, want PR 3", third.Documents)+ }+}++func TestBitbucketConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument(t *testing.T) {+ connector := mustBitbucketConnector(t, 1, "repo-a", "repo-b")+ connector.doJSON = bitbucketFixtureDoJSON(t)++ end := mustTime(t, "2026-01-04T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch first failed: %v", err)+ }+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:1" {+ t.Fatalf("first documents = %+v, want PR 1", first.Documents)+ }+ resumeCheckpoint := cloneBitbucketCheckpointWithoutSourceID(t, first.Checkpoint)++ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end, Resume: resumeCheckpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:2" {+ t.Fatalf("resume documents = %+v, want PR 2", second.Documents)+ }+ if second.Documents[0].SourceID == first.Documents[0].SourceID {+ t.Fatalf("committed document was redelivered: %s", second.Documents[0].SourceID)+ }+}++func TestBitbucketConnectorOpenSyncResumesAcrossPagedRepo(t *testing.T) {+ connector := mustBitbucketConnector(t, 1, "repo-a")+ connector.doJSON = bitbucketPagedFixtureDoJSON(t)++ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch first failed: %v", err)+ }+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:1" {+ t.Fatalf("first documents = %+v, want PR 1", first.Documents)+ }+ if first.Checkpoint == nil {+ t.Fatalf("first checkpoint is nil")+ }++ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:2" {+ t.Fatalf("resume documents = %+v, want PR 2", second.Documents)+ }+ if _, err = resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("resume EOF = %v", err)+ }+}++func TestBitbucketConnectorOpenPrune(t *testing.T) {+ connector := mustBitbucketConnector(t, 10, "repo-a", "repo-b")+ connector.doJSON = bitbucketFixtureDoJSON(t)++ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ got := []string{}+ for _, doc := range batch.Documents {+ got = append(got, doc.SourceID)+ }+ want := []string{+ "bitbucket:acme:repo-a:pr:1",+ "bitbucket:acme:repo-a:pr:2",+ "bitbucket:acme:repo-b:pr:3",+ }+ if len(got) != len(want) {+ t.Fatalf("ids len = %d, want %d: %v", len(got), len(want), got)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("ids[%d] = %q, want %q", i, got[i], want[i])+ }+ }+}++func TestBitbucketConnectorValidateConnectorSetting(t *testing.T) {+ for _, tc := range []struct {+ name string+ status int+ body string+ want string+ }{+ {name: "unauthorized", status: http.StatusUnauthorized, body: `{"message":"API Token provided has no Bitbucket scopes."}`, want: "Invalid or expired Bitbucket credentials (HTTP 401)."},+ {name: "forbidden", status: http.StatusForbidden, body: `{"message":"nope"}`, want: "HTTP 403"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ connector := &BitbucketConnector{+ workspace: "acme",+ email: "[email protected]",+ apiToken: "token",+ batchSize: 10,+ baseURL: "https://api.bitbucket.test",+ doJSON: func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ return nil, &bitbucketHTTPError{Status: tc.status, Body: tc.body}+ },+ }+ err := connector.ValidateConnectorSetting(context.Background(), nil)+ if err == nil || !strings.Contains(err.Error(), tc.want) {+ t.Fatalf("err = %v, want contains %q", err, tc.want)+ }+ })+ }+}++func TestBitbucketConnectorValidateMissingConfig(t *testing.T) {+ cases := []struct {+ name string+ config map[string]any+ want string+ }{+ {"missing workspace", map[string]any{+ "credentials": map[string]any{"bitbucket_api_token": "token"},+ }, "workspace"},+ {"missing email", map[string]any{+ "workspace": "acme",+ "credentials": map[string]any{"bitbucket_api_token": "token"},+ }, "bitbucket_account_email"},+ {"missing token", map[string]any{+ "workspace": "acme",+ "credentials": map[string]any{"bitbucket_account_email": "[email protected]"},+ }, "bitbucket_api_token"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ connector, err := NewBitbucketConnector(tc.config)+ if err != nil {+ t.Fatalf("NewBitbucketConnector failed: %v", err)+ }+ if err := connector.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), tc.want) {+ t.Fatalf("err = %v, want contains %q", err, tc.want)+ }+ })+ }+}++func TestBitbucketGetJSONRetriesOnTransientStatuses(t *testing.T) {+ originalAllowAny := utility.AllowAnyHostForTest+ originalTries := bitbucketRetryTries+ originalDelay := bitbucketRetryBaseDelay+ originalBackoff := bitbucketRetryBackoff+ originalMaxDelay := bitbucketRetryMaxDelay+ utility.AllowAnyHostForTest = true+ bitbucketRetryTries = 3+ bitbucketRetryBaseDelay = time.Millisecond+ bitbucketRetryBackoff = 1+ bitbucketRetryMaxDelay = 10 * time.Millisecond+ t.Cleanup(func() {+ utility.AllowAnyHostForTest = originalAllowAny+ bitbucketRetryTries = originalTries+ bitbucketRetryBaseDelay = originalDelay+ bitbucketRetryBackoff = originalBackoff+ bitbucketRetryMaxDelay = originalMaxDelay+ })++ var calls atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch calls.Add(1) {+ case 1:+ w.Header().Set("Retry-After", "0")+ w.WriteHeader(http.StatusTooManyRequests)+ case 2:+ w.WriteHeader(http.StatusServiceUnavailable)+ default:+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(`{"pagelen":1}`))+ }+ }))+ defer server.Close()++ connector := &BitbucketConnector{+ workspace: "acme",+ email: "[email protected]",+ apiToken: "token",+ baseURL: server.URL,+ }+ var page map[string]any+ if _, err := connector.getJSON(context.Background(), server.URL+"/repositories/acme", &page); err != nil {+ t.Fatalf("getJSON failed: %v", err)+ }+ if calls.Load() != 3 {+ t.Fatalf("calls = %d, want 3", calls.Load())+ }+}++func TestRegisterBuiltInsOpensBitbucket(t *testing.T) {+ registry := NewRegistry()+ RegisterBuiltIns(registry)+ connector, err := registry.OpenFromConfig("bitbucket", map[string]any{+ "workspace": "acme",+ "credentials": map[string]any{+ "bitbucket_account_email": "[email protected]",+ "bitbucket_api_token": "token",+ },+ })+ if err != nil {+ t.Fatalf("OpenFromConfig failed: %v", err)+ }+ if _, ok := connector.(*BitbucketConnector); !ok {+ t.Fatalf("connector type = %T, want *BitbucketConnector", connector)+ }+}++func mustBitbucketConnector(t *testing.T, batchSize int, repos ...string) *BitbucketConnector {+ t.Helper()+ connector, err := NewBitbucketConnector(map[string]any{+ "workspace": "acme",+ "repository_slugs": strings.Join(repos, ","),+ "batch_size": batchSize,+ "credentials": map[string]any{+ "bitbucket_account_email": "[email protected]",+ "bitbucket_api_token": "token",+ },+ })+ if err != nil {+ t.Fatalf("NewBitbucketConnector failed: %v", err)+ }+ connector.baseURL = "https://api.bitbucket.test"+ return connector+}++// bitbucketFixtureDoJSON returns fixture Bitbucket JSON transport.+func bitbucketFixtureDoJSON(t *testing.T) func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ t.Helper()+ return func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ parsed, err := url.Parse(apiURL)+ if err != nil {+ t.Fatalf("parse api url: %v", err)+ }+ path := parsed.Path+ var body string+ switch {+ case path == "/repositories/acme/repo-a/pullrequests":+ body = `{+ "values": [+ {+ "id": 1,+ "title": "Add Bitbucket sync",+ "description": "Sync PR body",+ "state": "OPEN",+ "author": {"display_name": "Alice"},+ "reviewers": [],+ "participants": [],+ "comment_count": 1,+ "task_count": 0,+ "created_on": "2026-01-01T00:00:00+00:00",+ "updated_on": "2026-01-03T00:00:00+00:00",+ "source": {"branch": {"name": "feature"}},+ "destination": {"branch": {"name": "main"}},+ "links": {"html": {"href": "https://bitbucket.org/acme/repo-a/pull-requests/1"}}+ },+ {+ "id": 2,+ "title": "Old PR",+ "description": "outside window",+ "state": "MERGED",+ "author": {"display_name": "Bob"},+ "reviewers": [],+ "participants": [],+ "comment_count": 0,+ "task_count": 0,+ "created_on": "2026-01-01T00:00:00+00:00",+ "updated_on": "2026-01-02T00:00:00+00:00",+ "source": {"branch": {"name": "old"}},+ "destination": {"branch": {"name": "main"}},+ "links": {"html": {"href": "https://bitbucket.org/acme/repo-a/pull-requests/2"}}+ }+ ]+ }`+ case path == "/repositories/acme/repo-b/pullrequests":+ body = `{+ "values": [+ {+ "id": 3,+ "title": "Second repo PR",+ "description": "Repo B body",+ "state": "OPEN",+ "author": {"display_name": "Bob"},+ "reviewers": [],+ "participants": [],+ "comment_count": 0,+ "task_count": 0,+ "created_on": "2026-01-03T00:00:00+00:00",+ "updated_on": "2026-01-03T12:00:00+00:00",+ "source": {"branch": {"name": "feature-b"}},+ "destination": {"branch": {"name": "main"}},+ "links": {"html": {"href": "https://bitbucket.org/acme/repo-b/pull-requests/3"}}+ }+ ]+ }`+ default:+ t.Fatalf("unexpected api path %s", path)+ }+ if err := json.Unmarshal([]byte(body), out); err != nil {+ t.Fatalf("decode fixture: %v", err)+ }+ return http.Header{}, nil+ }+}++func bitbucketPagedFixtureDoJSON(t *testing.T) func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ t.Helper()+ return func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ parsed, err := url.Parse(apiURL)+ if err != nil {+ t.Fatalf("parse api url: %v", err)+ }+ if parsed.Path != "/repositories/acme/repo-a/pullrequests" {+ t.Fatalf("unexpected api path %s", parsed.Path)+ }+ var body string+ if parsed.Query().Get("cursor") == "2" {+ body = `{+ "values": [+ {+ "id": 2,+ "title": "Second page PR",+ "state": "OPEN",+ "author": {"display_name": "Bob"},+ "updated_on": "2026-01-04T00:00:00+00:00"+ }+ ]+ }`+ } else {+ body = `{+ "next": "https://api.bitbucket.test/repositories/acme/repo-a/pullrequests?cursor=2",+ "values": [+ {+ "id": 1,+ "title": "First page PR",+ "state": "OPEN",+ "author": {"display_name": "Alice"},+ "updated_on": "2026-01-03T00:00:00+00:00"+ }+ ]+ }`+ }+ if err := json.Unmarshal([]byte(body), out); err != nil {+ t.Fatalf("decode fixture: %v", err)+ }+ return http.Header{}, nil+ }+}++func cloneBitbucketCheckpointWithoutSourceID(t *testing.T, checkpoint *SyncCheckpoint) *SyncCheckpoint {+ t.Helper()+ if checkpoint == nil {+ t.Fatalf("checkpoint is nil")+ }+ var cursor bitbucketSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ t.Fatalf("decode checkpoint cursor: %v", err)+ }+ cursor.SourceID = ""+ data, err := json.Marshal(cursor)+ if err != nil {+ t.Fatalf("encode checkpoint cursor: %v", err)+ }+ clone := *checkpoint+ clone.Cursor = string(data)+ clone.SourceID = ""+ return &clone+}diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 5327280214..a4e6cf7c30 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -28,6 +28,7 @@ import (// and the raw-config factory (used by the test-connection endpoint).func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "rss", NewRSSConnector)+ registerBuiltIn(registry, "bitbucket", NewBitbucketConnector)registerBuiltIn(registry, "github", NewGitHubConnector)registerBuiltIn(registry, "gitlab", NewGitlabConnector)registerBuiltIn(registry, "gmail", NewGmailConnector)