Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(syncer): add Bitbucket connector with checkpoint resume (#18487)

mkaaadd648d19137c7
diff --git a/internal/syncer/connector/bitbucket.go b/internal/syncer/connector/bitbucket.go
new file mode 100644
index 0000000000..c5bcdd2505
--- /dev/null
+++ b/internal/syncer/connector/bitbucket.go
@@ -0,0 +1,1053 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "regexp"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "ragflow/internal/utility"
+)
+
+const (
+ defaultBitbucketBatchSize = 32
+ bitbucketRequestTimeout = 60 * time.Second
+ bitbucketPRPageSize = 50
+ bitbucketRepoPageSize = 100
+ bitbucketBaseURL = "https://api.bitbucket.org/2.0"
+)
+
+var (
+ bitbucketRetryTries = 6
+ bitbucketRetryBaseDelay = 1 * time.Second
+ bitbucketRetryBackoff = 2.0
+ bitbucketRetryMaxDelay = 30 * time.Second
+ bitbucketSpaceRE = regexp.MustCompile(`\s+`)
+)
+
+const bitbucketPRFields = "next,page,pagelen,values.author,values.close_source_branch,values.closed_by,values.comment_count,values.created_on,values.description,values.destination,values.draft,values.id,values.links,values.merge_commit,values.participants,values.reason,values.rendered,values.reviewers,values.source,values.state,values.summary,values.task_count,values.title,values.type,values.updated_on"
+
+const bitbucketSlimPRFields = "next,page,pagelen,values.id"
+
+const bitbucketRepoFields = "next,page,pagelen,values.slug,values.full_name,values.project.key"
+
+// BitbucketConnector reads Bitbucket Cloud pull requests.
+type BitbucketConnector struct {
+ workspace string
+ repositorySlugs []string
+ projects []string
+ email string
+ apiToken string
+ batchSize int
+ baseURL string
+ doJSON func(ctx context.Context, apiURL string, out any) (http.Header, error)
+}
+
+// NewBitbucketConnector creates a Bitbucket connector from config.
+func NewBitbucketConnector(config map[string]any) (*BitbucketConnector, error) {
+ credentials, _ := config["credentials"].(map[string]any)
+ return &BitbucketConnector{
+ workspace: strings.TrimSpace(stringConfig(config["workspace"])),
+ repositorySlugs: splitBitbucketList(stringConfig(config["repository_slugs"])),
+ projects: splitBitbucketList(stringConfig(config["projects"])),
+ email: strings.TrimSpace(stringConfig(credentials["bitbucket_account_email"])),
+ apiToken: strings.TrimSpace(stringConfig(credentials["bitbucket_api_token"])),
+ batchSize: configInt(config["batch_size"], defaultBitbucketBatchSize),
+ baseURL: bitbucketBaseURL,
+ }, nil
+}
+
+// Validate validates Bitbucket connector settings and credentials.
+func (c *BitbucketConnector) Validate(ctx context.Context) error {
+ if c == nil {
+ return fmt.Errorf("bitbucket connector is nil")
+ }
+ if c.workspace == "" {
+ return fmt.Errorf("Invalid connector settings: 'workspace' must be provided")
+ }
+ if c.email == "" {
+ return fmt.Errorf("Missing bitbucket_account_email in credentials")
+ }
+ if c.apiToken == "" {
+ return fmt.Errorf("Missing bitbucket_api_token in credentials")
+ }
+ if c.batchSize <= 0 {
+ return fmt.Errorf("batch_size must be a positive integer")
+ }
+ repos, err := c.listRepos(ctx)
+ if err != nil {
+ return err
+ }
+ if len(repos) == 0 {
+ return fmt.Errorf("found no repositories for Bitbucket workspace %s", c.workspace)
+ }
+ return nil
+}
+
+// ValidateConnectorSetting validates Bitbucket settings from an unsaved config.
+func (c *BitbucketConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {
+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)
+ defer cancel()
+ if c == nil {
+ return fmt.Errorf("bitbucket connector is nil")
+ }
+ if c.workspace == "" {
+ return fmt.Errorf("Invalid connector settings: 'workspace' must be provided")
+ }
+ if c.email == "" {
+ return fmt.Errorf("Missing bitbucket_account_email in credentials")
+ }
+ if c.apiToken == "" {
+ return fmt.Errorf("Missing bitbucket_api_token in credentials")
+ }
+ var page map[string]any
+ _, err := c.getJSON(ctx, c.apiURL("/repositories/"+url.PathEscape(c.workspace), url.Values{
+ "pagelen": {"1"},
+ "fields": {"pagelen"},
+ }), &page)
+ if err != nil {
+ var httpErr *bitbucketHTTPError
+ if errors.As(err, &httpErr) {
+ switch httpErr.Status {
+ case http.StatusUnauthorized:
+ return fmt.Errorf("Invalid or expired Bitbucket credentials (HTTP 401).")
+ case http.StatusForbidden:
+ return fmt.Errorf("Insufficient permissions to access Bitbucket workspace (HTTP 403).")
+ }
+ return err
+ }
+ return err
+ }
+ return nil
+}
+
+// OpenSync opens one Bitbucket sync session.
+func (c *BitbucketConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {
+ repos, err := c.listRepos(ctx)
+ if err != nil {
+ return nil, err
+ }
+ if len(repos) == 0 {
+ return nil, fmt.Errorf("found no repositories for Bitbucket workspace %s", c.workspace)
+ }
+ session := &bitbucketSyncSession{
+ connector: c,
+ repos: repos,
+ batchSize: c.batchSize,
+ windowStart: request.WindowStart,
+ windowEnd: request.WindowEnd,
+ }
+ session.applyResume(request.Resume)
+ return session, nil
+}
+
+// OpenPrune opens one complete Bitbucket prune snapshot session.
+func (c *BitbucketConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {
+ repos, err := c.listRepos(ctx)
+ if err != nil {
+ return nil, err
+ }
+ if len(repos) == 0 {
+ return nil, fmt.Errorf("found no repositories for Bitbucket workspace %s", c.workspace)
+ }
+ return &bitbucketPruneSession{
+ connector: c,
+ repos: repos,
+ batchSize: c.batchSize,
+ }, nil
+}
+
+// listRepos returns deterministically ordered target repository slugs.
+func (c *BitbucketConnector) listRepos(ctx context.Context) ([]string, error) {
+ var repos []string
+ if len(c.repositorySlugs) > 0 {
+ repos = append(repos, c.repositorySlugs...)
+ } else if len(c.projects) > 0 {
+ for _, project := range c.projects {
+ projectRepos, err := c.listWorkspaceRepos(ctx, project)
+ if err != nil {
+ return nil, err
+ }
+ repos = append(repos, projectRepos...)
+ }
+ } else {
+ var err error
+ repos, err = c.listWorkspaceRepos(ctx, "")
+ if err != nil {
+ return nil, err
+ }
+ }
+ return uniqueSortedStrings(repos), nil
+}
+
+// listWorkspaceRepos lists workspace repositories, optionally filtered by project key.
+func (c *BitbucketConnector) listWorkspaceRepos(ctx context.Context, project string) ([]string, error) {
+ query := bitbucketRepoListQuery()
+ if project != "" {
+ query.Set("q", fmt.Sprintf("project.key=%q", project))
+ }
+ var repos []string
+ for pageURL := ""; ; {
+ if pageURL == "" {
+ pageURL = c.apiURL("/repositories/"+url.PathEscape(c.workspace), query)
+ }
+ var page bitbucketRepositoryPage
+ if _, err := c.getJSON(ctx, pageURL, &page); err != nil {
+ return nil, err
+ }
+ for _, repo := range page.Values {
+ if repo.Slug != "" {
+ repos = append(repos, repo.Slug)
+ }
+ }
+ if page.Next == "" || len(page.Values) == 0 {
+ break
+ }
+ if err := c.validateBitbucketHost(page.Next); err != nil {
+ return nil, err
+ }
+ pageURL = page.Next
+ }
+ return repos, nil
+}
+
+// listPullRequestPage returns one Bitbucket pull request page.
+func (c *BitbucketConnector) listPullRequestPage(ctx context.Context, repo, pageURL string, windowStart *time.Time, windowEnd time.Time) (bitbucketPullRequestPage, error) {
+ if pageURL == "" {
+ pageURL = c.apiURL("/repositories/"+url.PathEscape(c.workspace)+"/"+url.PathEscape(repo)+"/pullrequests", bitbucketPRListQuery(windowStart, windowEnd))
+ }
+ if err := c.validateBitbucketHost(pageURL); err != nil {
+ return bitbucketPullRequestPage{}, err
+ }
+ var page bitbucketPullRequestPage
+ if _, err := c.getJSON(ctx, pageURL, &page); err != nil {
+ return bitbucketPullRequestPage{}, err
+ }
+ return page, nil
+}
+
+// listSlimPullRequestPage returns one Bitbucket pull request page for prune snapshots.
+func (c *BitbucketConnector) listSlimPullRequestPage(ctx context.Context, repo, pageURL string) (bitbucketPullRequestPage, error) {
+ if pageURL == "" {
+ query := url.Values{
+ "fields": {bitbucketSlimPRFields},
+ "pagelen": {strconv.Itoa(bitbucketPRPageSize)},
+ "sort": {"updated_on"},
+ "q": {bitbucketPullRequestStateQuery()},
+ }
+ pageURL = c.apiURL("/repositories/"+url.PathEscape(c.workspace)+"/"+url.PathEscape(repo)+"/pullrequests", query)
+ }
+ if err := c.validateBitbucketHost(pageURL); err != nil {
+ return bitbucketPullRequestPage{}, err
+ }
+ var page bitbucketPullRequestPage
+ if _, err := c.getJSON(ctx, pageURL, &page); err != nil {
+ return bitbucketPullRequestPage{}, err
+ }
+ return page, nil
+}
+
+// validateBitbucketHost rejects a server-supplied pagination URL whose host does
+// not match the configured Bitbucket host, so authenticated requests cannot be
+// redirected at an attacker-controlled host.
+func (c *BitbucketConnector) validateBitbucketHost(rawURL string) error {
+ parsed, err := url.Parse(strings.TrimSpace(rawURL))
+ if err != nil {
+ return fmt.Errorf("bitbucket: invalid pagination URL: %w", err)
+ }
+ base, err := url.Parse(strings.TrimSpace(c.baseURL))
+ if err != nil {
+ return fmt.Errorf("bitbucket: invalid configured base URL: %w", err)
+ }
+ if !strings.EqualFold(parsed.Scheme, base.Scheme) {
+ return fmt.Errorf("bitbucket: pagination URL scheme %q does not match configured scheme %q", parsed.Scheme, base.Scheme)
+ }
+ if !strings.EqualFold(parsed.Hostname(), base.Hostname()) {
+ return fmt.Errorf("bitbucket: pagination URL host %q does not match configured host %q", parsed.Hostname(), base.Hostname())
+ }
+ if effectiveBitbucketPort(parsed) != effectiveBitbucketPort(base) {
+ return fmt.Errorf("bitbucket: pagination URL port %q does not match configured port %q", effectiveBitbucketPort(parsed), effectiveBitbucketPort(base))
+ }
+ return nil
+}
+
+// effectiveBitbucketPort returns the explicit port, falling back to the
+// scheme's default port so an explicit default port still matches.
+func effectiveBitbucketPort(u *url.URL) string {
+ if p := u.Port(); p != "" {
+ return p
+ }
+ switch strings.ToLower(u.Scheme) {
+ case "http":
+ return "80"
+ case "https":
+ return "443"
+ }
+ return ""
+}
+
+// apiURL builds a Bitbucket API URL.
+func (c *BitbucketConnector) apiURL(path string, query url.Values) string {
+ path = strings.TrimLeft(path, "/")
+ u := strings.TrimRight(c.baseURL, "/") + "/" + path
+ if len(query) == 0 {
+ return u
+ }
+ return u + "?" + query.Encode()
+}
+
+// getJSON fetches a Bitbucket API JSON response into out.
+func (c *BitbucketConnector) getJSON(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ if c.doJSON != nil {
+ return c.doJSON(ctx, apiURL, out)
+ }
+ return c.getJSONWithRetry(ctx, apiURL, out)
+}
+
+func (c *BitbucketConnector) getJSONWithRetry(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ delay := bitbucketRetryBaseDelay
+ var lastErr error
+ for attempt := 0; attempt < bitbucketRetryTries; attempt++ {
+ resp, err := c.get(ctx, apiURL)
+ if err != nil {
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ lastErr = err
+ if attempt == bitbucketRetryTries-1 {
+ break
+ }
+ if err := sleepFor(ctx, delay); err != nil {
+ return nil, err
+ }
+ delay = bitbucketNextDelay(delay)
+ continue
+ }
+
+ if resp.StatusCode == http.StatusTooManyRequests {
+ retryAfter := bitbucketRetryAfter(resp)
+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 4096))
+ resp.Body.Close()
+ lastErr = &bitbucketHTTPError{Status: resp.StatusCode}
+ if attempt == bitbucketRetryTries-1 {
+ break
+ }
+ wait := retryAfter
+ if wait <= 0 {
+ wait = delay
+ }
+ if err := sleepFor(ctx, wait); err != nil {
+ return nil, err
+ }
+ delay = bitbucketNextDelay(delay)
+ continue
+ }
+
+ if resp.StatusCode >= http.StatusInternalServerError {
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ resp.Body.Close()
+ lastErr = &bitbucketHTTPError{Status: resp.StatusCode, Body: string(body)}
+ if attempt == bitbucketRetryTries-1 {
+ break
+ }
+ if err := sleepFor(ctx, delay); err != nil {
+ return nil, err
+ }
+ delay = bitbucketNextDelay(delay)
+ continue
+ }
+
+ if resp.StatusCode >= http.StatusBadRequest {
+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
+ resp.Body.Close()
+ return nil, &bitbucketHTTPError{Status: resp.StatusCode, Body: string(body)}
+ }
+
+ headers := resp.Header.Clone()
+ if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
+ resp.Body.Close()
+ return nil, err
+ }
+ resp.Body.Close()
+ return headers, nil
+ }
+ return nil, lastErr
+}
+
+// get performs one SSRF-safe authenticated GET request.
+func (c *BitbucketConnector) get(ctx context.Context, apiURL string) (*http.Response, error) {
+ hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)
+ if err != nil {
+ return nil, err
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header.Set("Accept", "application/json")
+ req.SetBasicAuth(c.email, c.apiToken)
+ client := utility.PinnedHTTPClient(hostname, resolvedIP, bitbucketRequestTimeout)
+ return client.Do(req)
+}
+
+func sleepFor(ctx context.Context, duration time.Duration) error {
+ if duration <= 0 {
+ return nil
+ }
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-time.After(duration):
+ return nil
+ }
+}
+
+func bitbucketNextDelay(delay time.Duration) time.Duration {
+ next := time.Duration(float64(delay) * bitbucketRetryBackoff)
+ if next > bitbucketRetryMaxDelay {
+ return bitbucketRetryMaxDelay
+ }
+ return next
+}
+
+func bitbucketRetryAfter(resp *http.Response) time.Duration {
+ raw := strings.TrimSpace(resp.Header.Get("Retry-After"))
+ if raw == "" {
+ return 0
+ }
+ seconds, err := strconv.ParseFloat(raw, 64)
+ if err != nil || seconds < 0 {
+ return 0
+ }
+ wait := time.Duration(seconds * float64(time.Second))
+ if wait > bitbucketRetryMaxDelay {
+ return bitbucketRetryMaxDelay
+ }
+ return wait
+}
+
+func bitbucketPullRequestStateQuery() string {
+ return `(state = "OPEN" OR state = "MERGED" OR state = "DECLINED")`
+}
+
+func bitbucketPRListQuery(windowStart *time.Time, windowEnd time.Time) url.Values {
+ query := url.Values{
+ "fields": {bitbucketPRFields},
+ "pagelen": {strconv.Itoa(bitbucketPRPageSize)},
+ "sort": {"updated_on"},
+ }
+ stateQuery := bitbucketPullRequestStateQuery()
+ if windowStart != nil && !windowEnd.IsZero() {
+ query.Set("q", fmt.Sprintf("%s AND (updated_on > %q AND updated_on <= %q)", stateQuery, bitbucketISO(*windowStart), bitbucketISO(windowEnd)))
+ } else {
+ query.Set("q", stateQuery)
+ }
+ return query
+}
+
+func bitbucketRepoListQuery() url.Values {
+ return url.Values{
+ "fields": {bitbucketRepoFields},
+ "pagelen": {strconv.Itoa(bitbucketRepoPageSize)},
+ "sort": {"full_name"},
+ }
+}
+
+func bitbucketISO(t time.Time) string {
+ return t.UTC().Format(time.RFC3339Nano)
+}
+
+type bitbucketHTTPError struct {
+ Status int
+ Body string
+}
+
+func (e *bitbucketHTTPError) Error() string {
+ if strings.TrimSpace(e.Body) != "" {
+ return fmt.Sprintf("Bitbucket API returned HTTP %d: %s", e.Status, strings.TrimSpace(e.Body))
+ }
+ return fmt.Sprintf("Bitbucket API returned HTTP %d", e.Status)
+}
+
+type bitbucketSyncSession struct {
+ connector *BitbucketConnector
+ repos []string
+ repoIndex int
+ pageURL string
+ batchSize int
+ windowStart *time.Time
+ windowEnd time.Time
+ buffer []bitbucketBufferedDocument
+ resumeRepo string
+ resumePageURL string
+ resumeOffset int
+ resumeSource string
+}
+
+// NextBatch returns the next Bitbucket document batch.
+func (s *bitbucketSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {
+ documents := make([]SourceDocument, 0, s.batchSize)
+ var checkpoint *SyncCheckpoint
+ if len(s.buffer) > 0 {
+ n := s.batchSize
+ if n > len(s.buffer) {
+ n = len(s.buffer)
+ }
+ for _, buffered := range s.buffer[:n] {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
+ s.buffer = s.buffer[n:]
+ }
+
+ for len(documents) < s.batchSize {
+ if s.repoIndex >= len(s.repos) {
+ if len(documents) == 0 {
+ return SyncBatch{}, io.EOF
+ }
+ break
+ }
+ batch, err := s.nextDocumentPage(ctx)
+ if err != nil {
+ return SyncBatch{}, err
+ }
+ if len(batch) == 0 {
+ if s.repoIndex >= len(s.repos) && len(documents) == 0 {
+ return SyncBatch{}, io.EOF
+ }
+ continue
+ }
+ remaining := s.batchSize - len(documents)
+ if len(batch) > remaining {
+ for _, buffered := range batch[:remaining] {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
+ s.buffer = append(s.buffer, batch[remaining:]...)
+ break
+ }
+ for _, buffered := range batch {
+ documents = append(documents, buffered.document)
+ checkpoint = buffered.checkpoint
+ }
+ }
+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil
+}
+
+// Close closes the Bitbucket sync session.
+func (s *bitbucketSyncSession) Close() error {
+ return nil
+}
+
+func (s *bitbucketSyncSession) nextDocumentPage(ctx context.Context) ([]bitbucketBufferedDocument, error) {
+ if s.repoIndex >= len(s.repos) {
+ return nil, nil
+ }
+ repo := s.repos[s.repoIndex]
+ pageURL := s.pageURL
+ page, err := s.connector.listPullRequestPage(ctx, repo, pageURL, s.windowStart, s.windowEnd)
+ if err != nil {
+ return nil, err
+ }
+ documents := make([]bitbucketBufferedDocument, 0, len(page.Values))
+ pageOffset := 0
+ for _, pr := range page.Values {
+ updatedAt := parseBitbucketTime(pr.UpdatedOn)
+ if beforeOrAtWindowStart(updatedAt, s.windowStart) {
+ continue
+ }
+ if afterWindowEnd(updatedAt, s.windowEnd) {
+ continue
+ }
+ doc := pr.toSourceDocument(s.connector.workspace, repo)
+ pageOffset++
+ documents = append(documents, bitbucketBufferedDocument{
+ document: doc,
+ checkpoint: bitbucketSyncCheckpoint(repo, pageURL, pageOffset, doc),
+ offset: pageOffset,
+ sourceID: doc.SourceID,
+ })
+ }
+ documents = s.filterResumedDocuments(repo, pageURL, documents)
+ if page.Next == "" || len(page.Values) == 0 {
+ s.advanceRepo()
+ } else {
+ s.pageURL = page.Next
+ }
+ return documents, nil
+}
+
+// advanceRepo moves a Bitbucket sync session to the next repository.
+func (s *bitbucketSyncSession) advanceRepo() {
+ s.repoIndex++
+ s.pageURL = ""
+ s.clearResume()
+}
+
+// applyResume advances a sync session to the last committed Bitbucket position.
+func (s *bitbucketSyncSession) applyResume(checkpoint *SyncCheckpoint) {
+ if checkpoint == nil || checkpoint.Cursor == "" {
+ return
+ }
+ var cursor bitbucketSyncCursor
+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
+ return
+ }
+ if cursor.RepoSlug == "" {
+ return
+ }
+ for index, repo := range s.repos {
+ if repo != cursor.RepoSlug {
+ continue
+ }
+ s.repoIndex = index
+ s.pageURL = cursor.PageURL
+ s.resumeRepo = repo
+ s.resumePageURL = cursor.PageURL
+ s.resumeOffset = cursor.PageOffset
+ s.resumeSource = firstNonEmpty(cursor.SourceID, checkpoint.SourceID)
+ return
+ }
+}
+
+// filterResumedDocuments drops documents through the committed checkpoint.
+func (s *bitbucketSyncSession) filterResumedDocuments(repo, pageURL string, candidates []bitbucketBufferedDocument) []bitbucketBufferedDocument {
+ if s.resumeRepo == "" || repo != s.resumeRepo || pageURL != s.resumePageURL {
+ return candidates
+ }
+ if s.resumeSource != "" {
+ for index, candidate := range candidates {
+ if candidate.sourceID == s.resumeSource {
+ s.clearResume()
+ return candidates[index+1:]
+ }
+ }
+ }
+ if s.resumeOffset > 0 {
+ filtered := candidates[:0]
+ for _, candidate := range candidates {
+ if candidate.offset > s.resumeOffset {
+ filtered = append(filtered, candidate)
+ }
+ }
+ s.clearResume()
+ return filtered
+ }
+ s.clearResume()
+ return candidates
+}
+
+func (s *bitbucketSyncSession) clearResume() {
+ s.resumeRepo = ""
+ s.resumePageURL = ""
+ s.resumeOffset = 0
+ s.resumeSource = ""
+}
+
+type bitbucketPruneSession struct {
+ connector *BitbucketConnector
+ repos []string
+ repoIndex int
+ pageURL string
+ batchSize int
+ buffer []SlimDocument
+}
+
+// NextBatch returns the next Bitbucket prune snapshot batch.
+func (s *bitbucketPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {
+ documents := make([]SlimDocument, 0, s.batchSize)
+ if len(s.buffer) > 0 {
+ n := s.batchSize
+ if n > len(s.buffer) {
+ n = len(s.buffer)
+ }
+ documents = append(documents, s.buffer[:n]...)
+ s.buffer = s.buffer[n:]
+ }
+ for len(documents) < s.batchSize {
+ if s.repoIndex >= len(s.repos) {
+ if len(documents) == 0 {
+ return PruneBatch{}, io.EOF
+ }
+ break
+ }
+ batch, err := s.nextSlimPage(ctx)
+ if err != nil {
+ return PruneBatch{}, err
+ }
+ if len(batch) == 0 {
+ if s.repoIndex >= len(s.repos) && len(documents) == 0 {
+ return PruneBatch{}, io.EOF
+ }
+ continue
+ }
+ remaining := s.batchSize - len(documents)
+ if len(batch) > remaining {
+ documents = append(documents, batch[:remaining]...)
+ s.buffer = append(s.buffer, batch[remaining:]...)
+ break
+ }
+ documents = append(documents, batch...)
+ }
+ return PruneBatch{Documents: documents}, nil
+}
+
+// Close closes the Bitbucket prune session.
+func (s *bitbucketPruneSession) Close() error {
+ return nil
+}
+
+func (s *bitbucketPruneSession) nextSlimPage(ctx context.Context) ([]SlimDocument, error) {
+ if s.repoIndex >= len(s.repos) {
+ return nil, nil
+ }
+ repo := s.repos[s.repoIndex]
+ pageURL := s.pageURL
+ page, err := s.connector.listSlimPullRequestPage(ctx, repo, pageURL)
+ if err != nil {
+ return nil, err
+ }
+ documents := make([]SlimDocument, 0, len(page.Values))
+ for _, pr := range page.Values {
+ documents = append(documents, SlimDocument{SourceID: bitbucketSourceID(s.connector.workspace, repo, pr.ID)})
+ }
+ if page.Next == "" || len(page.Values) == 0 {
+ s.repoIndex++
+ s.pageURL = ""
+ } else {
+ s.pageURL = page.Next
+ }
+ return documents, nil
+}
+
+type bitbucketRepositoryPage struct {
+ Values []bitbucketRepository `json:"values"`
+ Next string `json:"next"`
+}
+
+type bitbucketRepository struct {
+ Slug string `json:"slug"`
+}
+
+type bitbucketPullRequestPage struct {
+ Values []bitbucketPullRequest `json:"values"`
+ Next string `json:"next"`
+}
+
+type bitbucketPullRequest struct {
+ ID int `json:"id"`
+ Title string `json:"title"`
+ Description string `json:"description"`
+ State string `json:"state"`
+ Reason string `json:"reason"`
+ Draft bool `json:"draft"`
+ Author *bitbucketUser `json:"author"`
+ Reviewers []bitbucketUser `json:"reviewers"`
+ Participants []bitbucketParticipant `json:"participants"`
+ CommentCount int `json:"comment_count"`
+ TaskCount int `json:"task_count"`
+ CreatedOn string `json:"created_on"`
+ UpdatedOn string `json:"updated_on"`
+ Source bitbucketBranchRef `json:"source"`
+ Destination bitbucketBranchRef `json:"destination"`
+ Links bitbucketPRLinks `json:"links"`
+ ClosedBy *bitbucketUser `json:"closed_by"`
+ CloseSourceBranch bool `json:"close_source_branch"`
+}
+
+type bitbucketUser struct {
+ DisplayName string `json:"display_name"`
+ Nickname string `json:"nickname"`
+}
+
+type bitbucketParticipant struct {
+ User bitbucketUser `json:"user"`
+ Approved bool `json:"approved"`
+}
+
+type bitbucketBranchRef struct {
+ Branch struct {
+ Name string `json:"name"`
+ } `json:"branch"`
+}
+
+type bitbucketPRLinks struct {
+ HTML struct {
+ Href string `json:"href"`
+ } `json:"html"`
+}
+
+type bitbucketSyncCursor struct {
+ RepoSlug string `json:"repo_slug"`
+ PageURL string `json:"page_url,omitempty"`
+ PageOffset int `json:"page_offset,omitempty"`
+ SourceID string `json:"source_id,omitempty"`
+}
+
+type bitbucketBufferedDocument struct {
+ document SourceDocument
+ checkpoint *SyncCheckpoint
+ offset int
+ sourceID string
+}
+
+func bitbucketSyncCheckpoint(repo, pageURL string, offset int, doc SourceDocument) *SyncCheckpoint {
+ cursor, err := json.Marshal(bitbucketSyncCursor{
+ RepoSlug: repo,
+ PageURL: pageURL,
+ PageOffset: offset,
+ SourceID: doc.SourceID,
+ })
+ if err != nil {
+ return nil
+ }
+ updatedAt := doc.UpdatedAt
+ return &SyncCheckpoint{
+ Cursor: string(cursor),
+ SourceID: doc.SourceID,
+ UpdatedAt: &updatedAt,
+ }
+}
+
+func (p bitbucketPullRequest) toSourceDocument(workspace, repo string) SourceDocument {
+ id := p.ID
+ title := strings.TrimSpace(p.Title)
+ if title == "" {
+ title = fmt.Sprintf("PR %d", id)
+ }
+ body := bitbucketPRBody(p, title)
+ updatedAt := parseBitbucketTime(p.UpdatedOn)
+ reviewers := bitbucketUserNames(p.Reviewers)
+ approvedBy := bitbucketApprovedBy(p.Participants)
+ author := bitbucketUserName(p.Author)
+ closedBy := bitbucketUserName(p.ClosedBy)
+ sourceBranch := p.Source.Branch.Name
+ destinationBranch := p.Destination.Branch.Name
+ link := p.Links.HTML.Href
+ if link == "" {
+ link = fmt.Sprintf("https://bitbucket.org/%s/%s/pull-requests/%d", workspace, repo, id)
+ }
+ return SourceDocument{
+ SourceID: bitbucketSourceID(workspace, repo, id),
+ SemanticIdentifier: bitbucketSemanticIdentifier(id, title),
+ Extension: ".md",
+ Blob: []byte(body),
+ UpdatedAt: updatedAt,
+ SizeBytes: int64(len(body)),
+ Metadata: map[string]any{
+ "object_type": "PullRequest",
+ "workspace": workspace,
+ "repository": repo,
+ "pr_key": fmt.Sprintf("%s/%s#%d", workspace, repo, id),
+ "id": strconv.Itoa(id),
+ "title": title,
+ "state": p.State,
+ "draft": strconv.FormatBool(p.Draft),
+ "link": link,
+ "author": author,
+ "reviewers": reviewers,
+ "approved_by": approvedBy,
+ "comment_count": strconv.Itoa(p.CommentCount),
+ "task_count": strconv.Itoa(p.TaskCount),
+ "created_on": p.CreatedOn,
+ "updated_on": p.UpdatedOn,
+ "source_branch": sourceBranch,
+ "destination_branch": destinationBranch,
+ "closed_by": closedBy,
+ "close_source_branch": strconv.FormatBool(p.CloseSourceBranch),
+ },
+ Fingerprint: stableFingerprint(map[string]any{
+ "object_type": "PullRequest",
+ "workspace": workspace,
+ "repository": repo,
+ "pr_id": id,
+ "title": title,
+ "description": p.Description,
+ "state": p.State,
+ "reason": p.Reason,
+ "draft": p.Draft,
+ "updated_on": p.UpdatedOn,
+ "created_on": p.CreatedOn,
+ "source_branch": sourceBranch,
+ "destination_branch": destinationBranch,
+ "author": author,
+ "reviewers": reviewers,
+ "approved_by": approvedBy,
+ "comment_count": p.CommentCount,
+ "task_count": p.TaskCount,
+ "closed_by": closedBy,
+ "close_source_branch": p.CloseSourceBranch,
+ }),
+ }
+}
+
+func bitbucketSourceID(workspace, repo string, id int) string {
+ return fmt.Sprintf("bitbucket:%s:%s:pr:%d", workspace, repo, id)
+}
+
+func bitbucketSemanticIdentifier(id int, title string) string {
+ return fmt.Sprintf("#%d: %s", id, sanitizeBitbucketName(title))
+}
+
+func bitbucketPRBody(p bitbucketPullRequest, title string) string {
+ var body strings.Builder
+ fmt.Fprintf(&body, "Pull Request Information:\n- Pull Request ID: %d\n- Title: %s\n- State: %s", p.ID, title, p.State)
+ if p.Draft {
+ body.WriteString(" (Draft)")
+ }
+ body.WriteString("\n")
+ if strings.EqualFold(p.State, "DECLINED") {
+ reason := strings.TrimSpace(p.Reason)
+ if reason == "" {
+ reason = "N/A"
+ }
+ fmt.Fprintf(&body, "- Reason: %s\n", reason)
+ }
+ fmt.Fprintf(&body, "- Author: %s\n", bitbucketBodyAuthor(p.Author))
+ reviewers := bitbucketUserNames(p.Reviewers)
+ if len(reviewers) == 0 {
+ body.WriteString("- Reviewers: N/A\n")
+ } else {
+ fmt.Fprintf(&body, "- Reviewers: %s\n", strings.Join(reviewers, ", "))
+ }
+ fmt.Fprintf(&body, "- Branch: %s -> %s\n", p.Source.Branch.Name, p.Destination.Branch.Name)
+ fmt.Fprintf(&body, "- Created: %s\n", bitbucketDateOrNA(p.CreatedOn))
+ fmt.Fprintf(&body, "- Updated: %s", bitbucketDateOrNA(p.UpdatedOn))
+ if description := strings.TrimSpace(p.Description); description != "" {
+ body.WriteString("\n\nDescription:\n")
+ body.WriteString(description)
+ }
+ return body.String()
+}
+
+func bitbucketBodyAuthor(user *bitbucketUser) string {
+ if user == nil {
+ return "N/A"
+ }
+ return bitbucketUserName(user)
+}
+
+func bitbucketUserName(user *bitbucketUser) string {
+ if user == nil {
+ return ""
+ }
+ if user.DisplayName != "" {
+ return user.DisplayName
+ }
+ if user.Nickname != "" {
+ return user.Nickname
+ }
+ return "unknown"
+}
+
+func bitbucketUserNames(users []bitbucketUser) []string {
+ out := make([]string, 0, len(users))
+ for i := range users {
+ out = append(out, bitbucketUserName(&users[i]))
+ }
+ sort.Strings(out)
+ return out
+}
+
+func bitbucketApprovedBy(participants []bitbucketParticipant) []string {
+ out := make([]string, 0, len(participants))
+ for _, participant := range participants {
+ if participant.Approved {
+ out = append(out, bitbucketUserName(&participant.User))
+ }
+ }
+ sort.Strings(out)
+ return out
+}
+
+func bitbucketDateOrNA(value string) string {
+ if value == "" {
+ return "N/A"
+ }
+ return strings.Split(value, "T")[0]
+}
+
+func parseBitbucketTime(value string) time.Time {
+ value = strings.TrimSpace(value)
+ if value == "" {
+ return time.Time{}
+ }
+ for _, layout := range []string{time.RFC3339Nano, "2006-01-02 15:04:05", "2006-01-02T15:04:05"} {
+ if parsed, err := time.Parse(layout, value); err == nil {
+ return parsed.UTC()
+ }
+ }
+ return time.Time{}
+}
+
+// sanitizeBitbucketName mirrors the Bitbucket PR filename cleanup semantics.
+func sanitizeBitbucketName(name string) string {
+ name = strings.TrimSpace(name)
+ if name == "" {
+ name = "PR"
+ }
+ replacer := strings.NewReplacer("\\", " ", "?", " ", "#", " ", "%", " ", "*", " ", ":", " ", "|", " ", "<", " ", ">", " ", `"`, " ")
+ name = replacer.Replace(name)
+ name = strings.ReplaceAll(name, "/", " ")
+ name = bitbucketSpaceRE.ReplaceAllString(name, " ")
+ name = strings.TrimSpace(name)
+ const maxNameRunes = 200
+ if runes := []rune(name); len(runes) > maxNameRunes {
+ name = strings.TrimSpace(string(runes[:maxNameRunes]))
+ }
+ if name == "" {
+ name = "PR"
+ }
+ return name + ".md"
+}
+
+func splitBitbucketList(value string) []string {
+ parts := strings.Split(value, ",")
+ out := make([]string, 0, len(parts))
+ for _, part := range parts {
+ part = strings.TrimSpace(part)
+ if part != "" {
+ out = append(out, part)
+ }
+ }
+ return out
+}
+
+func uniqueSortedStrings(values []string) []string {
+ if len(values) == 0 {
+ return []string{}
+ }
+ values = append([]string(nil), values...)
+ sort.Strings(values)
+ unique := values[:1]
+ for _, value := range values[1:] {
+ if value != unique[len(unique)-1] {
+ unique = append(unique, value)
+ }
+ }
+ return unique
+}
diff --git a/internal/syncer/connector/bitbucket_test.go b/internal/syncer/connector/bitbucket_test.go
new file mode 100644
index 0000000000..846184c10e
--- /dev/null
+++ b/internal/syncer/connector/bitbucket_test.go
@@ -0,0 +1,718 @@
+package connector
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "net/url"
+ "strings"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "ragflow/internal/utility"
+)
+
+func TestNewBitbucketConnectorParsesConfig(t *testing.T) {
+ connector, err := NewBitbucketConnector(map[string]any{
+ "workspace": " acme ",
+ "repository_slugs": "repo-a, repo-b,",
+ "projects": "P1, P2",
+ "batch_size": 7,
+ "credentials": map[string]any{
+ "bitbucket_account_email": "[email protected]",
+ "bitbucket_api_token": "token",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewBitbucketConnector failed: %v", err)
+ }
+ if connector.workspace != "acme" {
+ t.Fatalf("workspace = %q, want acme", connector.workspace)
+ }
+ if len(connector.repositorySlugs) != 2 || connector.repositorySlugs[0] != "repo-a" || connector.repositorySlugs[1] != "repo-b" {
+ t.Fatalf("repositorySlugs = %v", connector.repositorySlugs)
+ }
+ if len(connector.projects) != 2 || connector.projects[0] != "P1" || connector.projects[1] != "P2" {
+ t.Fatalf("projects = %v", connector.projects)
+ }
+ if connector.email != "[email protected]" || connector.apiToken != "token" {
+ t.Fatalf("credentials = %q %q", connector.email, connector.apiToken)
+ }
+ if connector.batchSize != 7 {
+ t.Fatalf("batch_size = %d, want 7", connector.batchSize)
+ }
+ if connector.baseURL != bitbucketBaseURL {
+ t.Fatalf("baseURL = %q, want %q", connector.baseURL, bitbucketBaseURL)
+ }
+}
+
+func TestBitbucketListReposUsesConfiguredSlugsBeforeProjects(t *testing.T) {
+ connector := &BitbucketConnector{
+ workspace: "acme",
+ repositorySlugs: []string{"repo-b", "repo-a", "repo-a"},
+ projects: []string{"P1"},
+ baseURL: "https://api.bitbucket.test",
+ doJSON: func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ t.Fatalf("repository slug config should not call the workspace API")
+ return nil, nil
+ },
+ }
+ repos, err := connector.listRepos(context.Background())
+ if err != nil {
+ t.Fatalf("listRepos failed: %v", err)
+ }
+ if len(repos) != 2 || repos[0] != "repo-a" || repos[1] != "repo-b" {
+ t.Fatalf("repos = %v, want [repo-a repo-b]", repos)
+ }
+}
+
+func TestBitbucketListReposFiltersByProjectKey(t *testing.T) {
+ connector := &BitbucketConnector{
+ workspace: "acme",
+ projects: []string{"P1", "P2"},
+ baseURL: "https://api.bitbucket.test",
+ doJSON: func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ parsed, err := url.Parse(apiURL)
+ if err != nil {
+ t.Fatalf("parse api url: %v", err)
+ }
+ if parsed.Path != "/repositories/acme" {
+ t.Fatalf("path = %q", parsed.Path)
+ }
+ project := parsed.Query().Get("q")
+ if project != `project.key="P1"` && project != `project.key="P2"` {
+ t.Fatalf("project query = %q", project)
+ }
+ slug := "repo-a"
+ if strings.Contains(project, "P2") {
+ slug = "repo-b"
+ }
+ body, _ := json.Marshal(bitbucketRepositoryPage{
+ Values: []bitbucketRepository{{Slug: slug}},
+ })
+ return nil, json.Unmarshal(body, out)
+ },
+ }
+ repos, err := connector.listRepos(context.Background())
+ if err != nil {
+ t.Fatalf("listRepos failed: %v", err)
+ }
+ if len(repos) != 2 || repos[0] != "repo-a" || repos[1] != "repo-b" {
+ t.Fatalf("repos = %v, want [repo-a repo-b]", repos)
+ }
+}
+
+func TestBitbucketPaginationRejectsForeignHost(t *testing.T) {
+ connector := &BitbucketConnector{
+ workspace: "acme",
+ baseURL: "https://api.bitbucket.test",
+ }
+
+ // listWorkspaceRepos refuses to follow a server-supplied next URL on a
+ // different host than the configured Bitbucket API.
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ body, _ := json.Marshal(bitbucketRepositoryPage{
+ Values: []bitbucketRepository{{Slug: "repo-a"}},
+ Next: "https://evil.example.com/repositories/acme",
+ })
+ return nil, json.Unmarshal(body, out)
+ }
+ if _, err := connector.listWorkspaceRepos(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "api.bitbucket.test") {
+ t.Fatalf("listWorkspaceRepos error = %v, want host mismatch", err)
+ }
+
+ // A same-host URL on a different scheme is still rejected.
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ body, _ := json.Marshal(bitbucketRepositoryPage{
+ Values: []bitbucketRepository{{Slug: "repo-a"}},
+ Next: "http://api.bitbucket.test/repositories/acme",
+ })
+ return nil, json.Unmarshal(body, out)
+ }
+ if _, err := connector.listWorkspaceRepos(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "scheme") {
+ t.Fatalf("listWorkspaceRepos error = %v, want scheme mismatch", err)
+ }
+
+ // A same-host URL on a different port is still rejected.
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ body, _ := json.Marshal(bitbucketRepositoryPage{
+ Values: []bitbucketRepository{{Slug: "repo-a"}},
+ Next: "https://api.bitbucket.test:8443/repositories/acme",
+ })
+ return nil, json.Unmarshal(body, out)
+ }
+ if _, err := connector.listWorkspaceRepos(context.Background(), ""); err == nil || !strings.Contains(err.Error(), "port") {
+ t.Fatalf("listWorkspaceRepos error = %v, want port mismatch", err)
+ }
+
+ // The pull-request page helpers stop before requesting a foreign host.
+ called := false
+ connector.doJSON = func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ called = true
+ return http.Header{}, nil
+ }
+ if _, err := connector.listPullRequestPage(context.Background(), "repo-a", "https://evil.example.com/repositories/acme/repo-a/pullrequests", nil, time.Time{}); err == nil {
+ t.Fatalf("listPullRequestPage should reject a foreign host")
+ }
+ if called {
+ t.Fatalf("listPullRequestPage requested a foreign host")
+ }
+
+ called = false
+ if _, err := connector.listSlimPullRequestPage(context.Background(), "repo-a", "https://evil.example.com/repositories/acme/repo-a/pullrequests"); err == nil {
+ t.Fatalf("listSlimPullRequestPage should reject a foreign host")
+ }
+ if called {
+ t.Fatalf("listSlimPullRequestPage requested a foreign host")
+ }
+}
+
+func TestBitbucketPRListQueryBuildsIncrementalWindow(t *testing.T) {
+ start := mustTime(t, "2026-01-02T12:00:00Z")
+ end := mustTime(t, "2026-01-04T00:00:00Z")
+ query := bitbucketPRListQuery(&start, end)
+ q := query.Get("q")
+ for _, want := range []string{
+ `(state = "OPEN" OR state = "MERGED" OR state = "DECLINED")`,
+ "updated_on >",
+ "updated_on <=",
+ } {
+ if !strings.Contains(q, want) {
+ t.Fatalf("q = %q, want contains %q", q, want)
+ }
+ }
+ if query.Get("sort") != "updated_on" {
+ t.Fatalf("sort = %q", query.Get("sort"))
+ }
+}
+
+func TestBitbucketPullRequestToSourceDocument(t *testing.T) {
+ pr := bitbucketPullRequest{
+ ID: 7,
+ Title: "Add: Bitbucket PR",
+ Description: "Sync PR body",
+ State: "MERGED",
+ Draft: true,
+ Author: &bitbucketUser{DisplayName: "Alice", Nickname: "alice"},
+ Reviewers: []bitbucketUser{
+ {DisplayName: "Zoe"},
+ {DisplayName: "Bob"},
+ },
+ Participants: []bitbucketParticipant{
+ {User: bitbucketUser{DisplayName: "Zoe"}, Approved: true},
+ {User: bitbucketUser{DisplayName: "Bill"}, Approved: false},
+ },
+ CommentCount: 3,
+ TaskCount: 2,
+ CreatedOn: "2026-01-01T10:00:00+00:00",
+ UpdatedOn: "2026-01-03T00:00:00+00:00",
+ Source: bitbucketBranchRef{Branch: struct {
+ Name string `json:"name"`
+ }{Name: "feature"}},
+ Destination: bitbucketBranchRef{Branch: struct {
+ Name string `json:"name"`
+ }{Name: "main"}},
+ Links: bitbucketPRLinks{HTML: struct {
+ Href string `json:"href"`
+ }{Href: "https://bitbucket.org/acme/repo/pull-requests/7"}},
+ }
+ first := pr.toSourceDocument("acme", "repo")
+ second := pr.toSourceDocument("acme", "repo")
+ if first.SourceID != "bitbucket:acme:repo:pr:7" {
+ t.Fatalf("source id = %q", first.SourceID)
+ }
+ if first.SemanticIdentifier != "#7: Add Bitbucket PR.md" {
+ t.Fatalf("semantic identifier = %q", first.SemanticIdentifier)
+ }
+ if first.Extension != ".md" {
+ t.Fatalf("extension = %q", first.Extension)
+ }
+ if first.Fingerprint == "" || first.Fingerprint != second.Fingerprint {
+ t.Fatalf("fingerprint unstable: %q %q", first.Fingerprint, second.Fingerprint)
+ }
+ if !strings.Contains(string(first.Blob), "Pull Request Information") ||
+ !strings.Contains(string(first.Blob), "Description:\nSync PR body") {
+ t.Fatalf("blob missing PR content: %s", first.Blob)
+ }
+ reviewers, _ := first.Metadata["reviewers"].([]string)
+ if len(reviewers) != 2 || reviewers[0] != "Bob" || reviewers[1] != "Zoe" {
+ t.Fatalf("reviewers = %v", reviewers)
+ }
+
+ changed := pr
+ changed.Title = "Add Bitbucket PR v2"
+ if got := changed.toSourceDocument("acme", "repo").Fingerprint; got == first.Fingerprint {
+ t.Fatalf("fingerprint did not change after title update")
+ }
+}
+
+func TestBitbucketConnectorOpenSyncUsesWindowAndFingerprint(t *testing.T) {
+ connector := mustBitbucketConnector(t, 1, "repo-a", "repo-b")
+ connector.doJSON = bitbucketFixtureDoJSON(t)
+
+ start := mustTime(t, "2026-01-02T12:00:00Z")
+ end := mustTime(t, "2026-01-04T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ if len(batch.Documents) != 1 {
+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))
+ }
+ doc := batch.Documents[0]
+ if doc.SourceID != "bitbucket:acme:repo-a:pr:1" {
+ t.Fatalf("source id = %q", doc.SourceID)
+ }
+ if doc.Fingerprint == "" {
+ t.Fatalf("fingerprint is empty")
+ }
+ second, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch second failed: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "bitbucket:acme:repo-b:pr:3" {
+ t.Fatalf("second documents = %+v, want PR 3", second.Documents)
+ }
+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("NextBatch EOF = %v", err)
+ }
+}
+
+func TestBitbucketConnectorOpenSyncResumesAfterCheckpoint(t *testing.T) {
+ connector := mustBitbucketConnector(t, 1, "repo-a", "repo-b")
+ connector.doJSON = bitbucketFixtureDoJSON(t)
+
+ end := mustTime(t, "2026-01-04T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch first failed: %v", err)
+ }
+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:1" {
+ t.Fatalf("first documents = %+v, want PR 1", first.Documents)
+ }
+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "bitbucket:acme:repo-a:pr:1" {
+ t.Fatalf("first checkpoint = %+v, want PR 1", first.Checkpoint)
+ }
+
+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end, Resume: first.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync failed: %v", err)
+ }
+ second, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resume NextBatch failed: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:2" {
+ t.Fatalf("resume documents = %+v, want PR 2", second.Documents)
+ }
+ if second.Checkpoint == nil || second.Checkpoint.SourceID != "bitbucket:acme:repo-a:pr:2" {
+ t.Fatalf("resume checkpoint = %+v, want PR 2", second.Checkpoint)
+ }
+ third, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resume NextBatch third failed: %v", err)
+ }
+ if len(third.Documents) != 1 || third.Documents[0].SourceID != "bitbucket:acme:repo-b:pr:3" {
+ t.Fatalf("resume documents = %+v, want PR 3", third.Documents)
+ }
+}
+
+func TestBitbucketConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument(t *testing.T) {
+ connector := mustBitbucketConnector(t, 1, "repo-a", "repo-b")
+ connector.doJSON = bitbucketFixtureDoJSON(t)
+
+ end := mustTime(t, "2026-01-04T00:00:00Z")
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch first failed: %v", err)
+ }
+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:1" {
+ t.Fatalf("first documents = %+v, want PR 1", first.Documents)
+ }
+ resumeCheckpoint := cloneBitbucketCheckpointWithoutSourceID(t, first.Checkpoint)
+
+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end, Resume: resumeCheckpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync failed: %v", err)
+ }
+ second, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resume NextBatch failed: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:2" {
+ t.Fatalf("resume documents = %+v, want PR 2", second.Documents)
+ }
+ if second.Documents[0].SourceID == first.Documents[0].SourceID {
+ t.Fatalf("committed document was redelivered: %s", second.Documents[0].SourceID)
+ }
+}
+
+func TestBitbucketConnectorOpenSyncResumesAcrossPagedRepo(t *testing.T) {
+ connector := mustBitbucketConnector(t, 1, "repo-a")
+ connector.doJSON = bitbucketPagedFixtureDoJSON(t)
+
+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true})
+ if err != nil {
+ t.Fatalf("OpenSync failed: %v", err)
+ }
+ first, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch first failed: %v", err)
+ }
+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:1" {
+ t.Fatalf("first documents = %+v, want PR 1", first.Documents)
+ }
+ if first.Checkpoint == nil {
+ t.Fatalf("first checkpoint is nil")
+ }
+
+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, Resume: first.Checkpoint})
+ if err != nil {
+ t.Fatalf("resume OpenSync failed: %v", err)
+ }
+ second, err := resumed.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("resume NextBatch failed: %v", err)
+ }
+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "bitbucket:acme:repo-a:pr:2" {
+ t.Fatalf("resume documents = %+v, want PR 2", second.Documents)
+ }
+ if _, err = resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {
+ t.Fatalf("resume EOF = %v", err)
+ }
+}
+
+func TestBitbucketConnectorOpenPrune(t *testing.T) {
+ connector := mustBitbucketConnector(t, 10, "repo-a", "repo-b")
+ connector.doJSON = bitbucketFixtureDoJSON(t)
+
+ session, err := connector.OpenPrune(context.Background(), PruneRequest{})
+ if err != nil {
+ t.Fatalf("OpenPrune failed: %v", err)
+ }
+ batch, err := session.NextBatch(context.Background())
+ if err != nil {
+ t.Fatalf("NextBatch failed: %v", err)
+ }
+ got := []string{}
+ for _, doc := range batch.Documents {
+ got = append(got, doc.SourceID)
+ }
+ want := []string{
+ "bitbucket:acme:repo-a:pr:1",
+ "bitbucket:acme:repo-a:pr:2",
+ "bitbucket:acme:repo-b:pr:3",
+ }
+ if len(got) != len(want) {
+ t.Fatalf("ids len = %d, want %d: %v", len(got), len(want), got)
+ }
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("ids[%d] = %q, want %q", i, got[i], want[i])
+ }
+ }
+}
+
+func TestBitbucketConnectorValidateConnectorSetting(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ status int
+ body string
+ want string
+ }{
+ {name: "unauthorized", status: http.StatusUnauthorized, body: `{"message":"API Token provided has no Bitbucket scopes."}`, want: "Invalid or expired Bitbucket credentials (HTTP 401)."},
+ {name: "forbidden", status: http.StatusForbidden, body: `{"message":"nope"}`, want: "HTTP 403"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ connector := &BitbucketConnector{
+ workspace: "acme",
+ email: "[email protected]",
+ apiToken: "token",
+ batchSize: 10,
+ baseURL: "https://api.bitbucket.test",
+ doJSON: func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ return nil, &bitbucketHTTPError{Status: tc.status, Body: tc.body}
+ },
+ }
+ err := connector.ValidateConnectorSetting(context.Background(), nil)
+ if err == nil || !strings.Contains(err.Error(), tc.want) {
+ t.Fatalf("err = %v, want contains %q", err, tc.want)
+ }
+ })
+ }
+}
+
+func TestBitbucketConnectorValidateMissingConfig(t *testing.T) {
+ cases := []struct {
+ name string
+ config map[string]any
+ want string
+ }{
+ {"missing workspace", map[string]any{
+ "credentials": map[string]any{"bitbucket_api_token": "token"},
+ }, "workspace"},
+ {"missing email", map[string]any{
+ "workspace": "acme",
+ "credentials": map[string]any{"bitbucket_api_token": "token"},
+ }, "bitbucket_account_email"},
+ {"missing token", map[string]any{
+ "workspace": "acme",
+ "credentials": map[string]any{"bitbucket_account_email": "[email protected]"},
+ }, "bitbucket_api_token"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ connector, err := NewBitbucketConnector(tc.config)
+ if err != nil {
+ t.Fatalf("NewBitbucketConnector failed: %v", err)
+ }
+ if err := connector.Validate(context.Background()); err == nil || !strings.Contains(err.Error(), tc.want) {
+ t.Fatalf("err = %v, want contains %q", err, tc.want)
+ }
+ })
+ }
+}
+
+func TestBitbucketGetJSONRetriesOnTransientStatuses(t *testing.T) {
+ originalAllowAny := utility.AllowAnyHostForTest
+ originalTries := bitbucketRetryTries
+ originalDelay := bitbucketRetryBaseDelay
+ originalBackoff := bitbucketRetryBackoff
+ originalMaxDelay := bitbucketRetryMaxDelay
+ utility.AllowAnyHostForTest = true
+ bitbucketRetryTries = 3
+ bitbucketRetryBaseDelay = time.Millisecond
+ bitbucketRetryBackoff = 1
+ bitbucketRetryMaxDelay = 10 * time.Millisecond
+ t.Cleanup(func() {
+ utility.AllowAnyHostForTest = originalAllowAny
+ bitbucketRetryTries = originalTries
+ bitbucketRetryBaseDelay = originalDelay
+ bitbucketRetryBackoff = originalBackoff
+ bitbucketRetryMaxDelay = originalMaxDelay
+ })
+
+ var calls atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch calls.Add(1) {
+ case 1:
+ w.Header().Set("Retry-After", "0")
+ w.WriteHeader(http.StatusTooManyRequests)
+ case 2:
+ w.WriteHeader(http.StatusServiceUnavailable)
+ default:
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"pagelen":1}`))
+ }
+ }))
+ defer server.Close()
+
+ connector := &BitbucketConnector{
+ workspace: "acme",
+ email: "[email protected]",
+ apiToken: "token",
+ baseURL: server.URL,
+ }
+ var page map[string]any
+ if _, err := connector.getJSON(context.Background(), server.URL+"/repositories/acme", &page); err != nil {
+ t.Fatalf("getJSON failed: %v", err)
+ }
+ if calls.Load() != 3 {
+ t.Fatalf("calls = %d, want 3", calls.Load())
+ }
+}
+
+func TestRegisterBuiltInsOpensBitbucket(t *testing.T) {
+ registry := NewRegistry()
+ RegisterBuiltIns(registry)
+ connector, err := registry.OpenFromConfig("bitbucket", map[string]any{
+ "workspace": "acme",
+ "credentials": map[string]any{
+ "bitbucket_account_email": "[email protected]",
+ "bitbucket_api_token": "token",
+ },
+ })
+ if err != nil {
+ t.Fatalf("OpenFromConfig failed: %v", err)
+ }
+ if _, ok := connector.(*BitbucketConnector); !ok {
+ t.Fatalf("connector type = %T, want *BitbucketConnector", connector)
+ }
+}
+
+func mustBitbucketConnector(t *testing.T, batchSize int, repos ...string) *BitbucketConnector {
+ t.Helper()
+ connector, err := NewBitbucketConnector(map[string]any{
+ "workspace": "acme",
+ "repository_slugs": strings.Join(repos, ","),
+ "batch_size": batchSize,
+ "credentials": map[string]any{
+ "bitbucket_account_email": "[email protected]",
+ "bitbucket_api_token": "token",
+ },
+ })
+ if err != nil {
+ t.Fatalf("NewBitbucketConnector failed: %v", err)
+ }
+ connector.baseURL = "https://api.bitbucket.test"
+ return connector
+}
+
+// bitbucketFixtureDoJSON returns fixture Bitbucket JSON transport.
+func bitbucketFixtureDoJSON(t *testing.T) func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ t.Helper()
+ return func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ parsed, err := url.Parse(apiURL)
+ if err != nil {
+ t.Fatalf("parse api url: %v", err)
+ }
+ path := parsed.Path
+ var body string
+ switch {
+ case path == "/repositories/acme/repo-a/pullrequests":
+ body = `{
+ "values": [
+ {
+ "id": 1,
+ "title": "Add Bitbucket sync",
+ "description": "Sync PR body",
+ "state": "OPEN",
+ "author": {"display_name": "Alice"},
+ "reviewers": [],
+ "participants": [],
+ "comment_count": 1,
+ "task_count": 0,
+ "created_on": "2026-01-01T00:00:00+00:00",
+ "updated_on": "2026-01-03T00:00:00+00:00",
+ "source": {"branch": {"name": "feature"}},
+ "destination": {"branch": {"name": "main"}},
+ "links": {"html": {"href": "https://bitbucket.org/acme/repo-a/pull-requests/1"}}
+ },
+ {
+ "id": 2,
+ "title": "Old PR",
+ "description": "outside window",
+ "state": "MERGED",
+ "author": {"display_name": "Bob"},
+ "reviewers": [],
+ "participants": [],
+ "comment_count": 0,
+ "task_count": 0,
+ "created_on": "2026-01-01T00:00:00+00:00",
+ "updated_on": "2026-01-02T00:00:00+00:00",
+ "source": {"branch": {"name": "old"}},
+ "destination": {"branch": {"name": "main"}},
+ "links": {"html": {"href": "https://bitbucket.org/acme/repo-a/pull-requests/2"}}
+ }
+ ]
+ }`
+ case path == "/repositories/acme/repo-b/pullrequests":
+ body = `{
+ "values": [
+ {
+ "id": 3,
+ "title": "Second repo PR",
+ "description": "Repo B body",
+ "state": "OPEN",
+ "author": {"display_name": "Bob"},
+ "reviewers": [],
+ "participants": [],
+ "comment_count": 0,
+ "task_count": 0,
+ "created_on": "2026-01-03T00:00:00+00:00",
+ "updated_on": "2026-01-03T12:00:00+00:00",
+ "source": {"branch": {"name": "feature-b"}},
+ "destination": {"branch": {"name": "main"}},
+ "links": {"html": {"href": "https://bitbucket.org/acme/repo-b/pull-requests/3"}}
+ }
+ ]
+ }`
+ default:
+ t.Fatalf("unexpected api path %s", path)
+ }
+ if err := json.Unmarshal([]byte(body), out); err != nil {
+ t.Fatalf("decode fixture: %v", err)
+ }
+ return http.Header{}, nil
+ }
+}
+
+func bitbucketPagedFixtureDoJSON(t *testing.T) func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ t.Helper()
+ return func(ctx context.Context, apiURL string, out any) (http.Header, error) {
+ parsed, err := url.Parse(apiURL)
+ if err != nil {
+ t.Fatalf("parse api url: %v", err)
+ }
+ if parsed.Path != "/repositories/acme/repo-a/pullrequests" {
+ t.Fatalf("unexpected api path %s", parsed.Path)
+ }
+ var body string
+ if parsed.Query().Get("cursor") == "2" {
+ body = `{
+ "values": [
+ {
+ "id": 2,
+ "title": "Second page PR",
+ "state": "OPEN",
+ "author": {"display_name": "Bob"},
+ "updated_on": "2026-01-04T00:00:00+00:00"
+ }
+ ]
+ }`
+ } else {
+ body = `{
+ "next": "https://api.bitbucket.test/repositories/acme/repo-a/pullrequests?cursor=2",
+ "values": [
+ {
+ "id": 1,
+ "title": "First page PR",
+ "state": "OPEN",
+ "author": {"display_name": "Alice"},
+ "updated_on": "2026-01-03T00:00:00+00:00"
+ }
+ ]
+ }`
+ }
+ if err := json.Unmarshal([]byte(body), out); err != nil {
+ t.Fatalf("decode fixture: %v", err)
+ }
+ return http.Header{}, nil
+ }
+}
+
+func cloneBitbucketCheckpointWithoutSourceID(t *testing.T, checkpoint *SyncCheckpoint) *SyncCheckpoint {
+ t.Helper()
+ if checkpoint == nil {
+ t.Fatalf("checkpoint is nil")
+ }
+ var cursor bitbucketSyncCursor
+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {
+ t.Fatalf("decode checkpoint cursor: %v", err)
+ }
+ cursor.SourceID = ""
+ data, err := json.Marshal(cursor)
+ if err != nil {
+ t.Fatalf("encode checkpoint cursor: %v", err)
+ }
+ clone := *checkpoint
+ clone.Cursor = string(data)
+ clone.SourceID = ""
+ return &clone
+}
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.go
index 5327280214..a4e6cf7c30 100644
--- a/internal/syncer/connector/builtin.go
+++ b/internal/syncer/connector/builtin.go
@@ -28,6 +28,7 @@ import (
// and the raw-config factory (used by the test-connection endpoint).
func RegisterBuiltIns(registry *Registry) {
registerBuiltIn(registry, "rss", NewRSSConnector)
+ registerBuiltIn(registry, "bitbucket", NewBitbucketConnector)
registerBuiltIn(registry, "github", NewGitHubConnector)
registerBuiltIn(registry, "gitlab", NewGitlabConnector)
registerBuiltIn(registry, "gmail", NewGmailConnector)