RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(go-syncer): add GitLab datasource connector with checkpoint resume (#18425)
mkaaada4ec4f84cf73
diff --git a/internal/syncer/connector/builtin.go b/internal/syncer/connector/builtin.goindex 9e8256b0b2..5327280214 100644--- a/internal/syncer/connector/builtin.go+++ b/internal/syncer/connector/builtin.go@@ -29,6 +29,7 @@ import (func RegisterBuiltIns(registry *Registry) {registerBuiltIn(registry, "rss", NewRSSConnector)registerBuiltIn(registry, "github", NewGitHubConnector)+ registerBuiltIn(registry, "gitlab", NewGitlabConnector)registerBuiltIn(registry, "gmail", NewGmailConnector)registerBuiltIn(registry, "google-drive", NewGoogleDriveConnector)registerBuiltIn(registry, "google_drive", NewGoogleDriveConnector)diff --git a/internal/syncer/connector/gitlab.go b/internal/syncer/connector/gitlab.gonew file mode 100644index 0000000000..12a783951f--- /dev/null+++ b/internal/syncer/connector/gitlab.go@@ -0,0 +1,1052 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package connector++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "path/filepath"+ "strconv"+ "strings"+ "time"++ "ragflow/internal/utility"+)++const (+ defaultGitlabBatchSize = 32+ gitlabRequestTimeout = 60 * time.Second+)++var gitlabExcludePatterns = []string{"logs", ".github", ".gitlab"}++// GitlabConnector reads GitLab merge requests, issues, and code files.+type GitlabConnector struct {+ projectOwner string+ projectName string+ gitlabURL string+ token string+ batchSize int+ includeMRs bool+ includeIssues bool+ includeCodeFiles bool+ baseURL string+ doJSON func(ctx context.Context, apiURL string, out any) (http.Header, error)+ doRaw func(ctx context.Context, apiURL string) ([]byte, error)+}++// NewGitlabConnector creates a GitLab connector from config.+func NewGitlabConnector(config map[string]any) (*GitlabConnector, error) {+ credentials, _ := config["credentials"].(map[string]any)+ token, _ := credentials["gitlab_access_token"].(string)+ gitlabURL := strings.TrimRight(stringConfig(config["gitlab_url"]), "/")+ if gitlabURL == "" {+ gitlabURL = "https://gitlab.com"+ }+ return &GitlabConnector{+ projectOwner: strings.TrimSpace(stringConfig(config["project_owner"])),+ projectName: strings.TrimSpace(stringConfig(config["project_name"])),+ gitlabURL: gitlabURL,+ token: strings.TrimSpace(token),+ batchSize: configInt(config["batch_size"], defaultGitlabBatchSize),+ includeMRs: configBoolDefault(config["include_mrs"], true),+ includeIssues: configBoolDefault(config["include_issues"], true),+ includeCodeFiles: configBoolDefault(config["include_code_files"], true),+ baseURL: gitlabURL + "/api/v4",+ }, nil+}++// Validate validates GitLab connector settings and credentials.+func (c *GitlabConnector) Validate(ctx context.Context) error {+ if c == nil {+ return fmt.Errorf("gitlab connector is nil")+ }+ if c.projectOwner == "" {+ return fmt.Errorf("Invalid connector settings: 'project_owner' must be provided")+ }+ if c.projectName == "" {+ return fmt.Errorf("Invalid connector settings: 'project_name' must be provided")+ }+ if c.token == "" {+ return fmt.Errorf("Missing gitlab_access_token in credentials")+ }+ if c.batchSize <= 0 {+ return fmt.Errorf("batch_size must be a positive integer")+ }+ if _, err := c.getProject(ctx); err != nil {+ return err+ }+ return nil+}++// ValidateConnectorSetting validates GitLab settings from an unsaved config.+func (c *GitlabConnector) ValidateConnectorSetting(ctx context.Context, request map[string]any) error {+ ctx, cancel := context.WithTimeout(ctx, connectorSettingValidationTimeout)+ defer cancel()+ if c == nil {+ return fmt.Errorf("gitlab connector is nil")+ }+ if c.projectOwner == "" {+ return fmt.Errorf("Invalid connector settings: 'project_owner' must be provided")+ }+ if c.projectName == "" {+ return fmt.Errorf("Invalid connector settings: 'project_name' must be provided")+ }+ if c.token == "" {+ return fmt.Errorf("Missing gitlab_access_token in credentials")+ }+ if c.batchSize <= 0 {+ return fmt.Errorf("batch_size must be a positive integer")+ }+ var user map[string]any+ if _, err := c.getJSON(ctx, c.apiURL("/user", nil), &user); err != nil {+ return err+ }+ return nil+}++// OpenSync opens one GitLab sync session.+func (c *GitlabConnector) OpenSync(ctx context.Context, request SyncRequest) (SyncSession, error) {+ project, err := c.getProject(ctx)+ if err != nil {+ return nil, err+ }+ session := &gitlabSyncSession{+ connector: c,+ project: project,+ batchSize: c.batchSize,+ stage: gitlabStageCodeFiles,+ treeQueue: []string{""},+ treePage: 1,+ page: 1,+ windowStart: request.WindowStart,+ windowEnd: request.WindowEnd,+ }+ if !c.includeCodeFiles {+ session.stage = gitlabStageMRs+ session.treeQueue = nil+ }+ session.applyResume(request.Resume)+ return session, nil+}++// OpenPrune opens one complete GitLab prune snapshot session.+func (c *GitlabConnector) OpenPrune(ctx context.Context, request PruneRequest) (PruneSession, error) {+ project, err := c.getProject(ctx)+ if err != nil {+ return nil, err+ }+ session := &gitlabPruneSession{+ connector: c,+ project: project,+ batchSize: c.batchSize,+ stage: gitlabStageCodeFiles,+ treeQueue: []string{""},+ treePage: 1,+ page: 1,+ }+ if !c.includeCodeFiles {+ session.stage = gitlabStageMRs+ session.treeQueue = nil+ }+ return session, nil+}++// getProject fetches the GitLab project info.+func (c *GitlabConnector) getProject(ctx context.Context) (gitlabProject, error) {+ encoded := gitlabProjectPath(c.projectOwner, c.projectName)+ var project gitlabProject+ if _, err := c.getJSON(ctx, c.apiURL("/projects/"+encoded, nil), &project); err != nil {+ return gitlabProject{}, err+ }+ return project, nil+}++// listMergeRequestPage returns one page of GitLab merge requests.+func (c *GitlabConnector) listMergeRequestPage(ctx context.Context, projectID int, page, pageSize int, windowStart *time.Time, windowEnd time.Time) ([]gitlabBufferedDocument, bool, error) {+ query := gitlabListQuery(page, pageSize)+ query.Set("order_by", "updated_at")+ query.Set("sort", "desc")+ var batch []gitlabMergeRequest+ headers, err := c.getJSON(ctx, c.apiURL(fmt.Sprintf("/projects/%d/merge_requests", projectID), query), &batch)+ if err != nil {+ return nil, false, err+ }+ documents := make([]gitlabBufferedDocument, 0, len(batch))+ doneByWindow := false+ pageOffset := 0+ for _, mr := range batch {+ if beforeOrAtWindowStart(mr.UpdatedAt, windowStart) {+ doneByWindow = true+ break+ }+ if afterWindowEnd(mr.UpdatedAt, windowEnd) {+ continue+ }+ doc := mr.toSourceDocument()+ pageOffset+++ documents = append(documents, gitlabBufferedDocument{+ document: doc,+ checkpoint: gitlabSyncCheckpoint(gitlabSyncCursor{Stage: gitlabStageMRs, Page: page, Offset: pageOffset, SourceID: doc.SourceID}, doc),+ offset: pageOffset,+ sourceID: doc.SourceID,+ })+ }+ done := doneByWindow || !gitlabHasNextPage(headers) || len(batch) == 0+ return documents, done, nil+}++// listIssuePage returns one page of GitLab issues.+func (c *GitlabConnector) listIssuePage(ctx context.Context, projectID int, page, pageSize int, windowStart *time.Time, windowEnd time.Time) ([]gitlabBufferedDocument, bool, error) {+ query := gitlabListQuery(page, pageSize)+ query.Set("order_by", "updated_at")+ query.Set("sort", "desc")+ var batch []gitlabIssue+ headers, err := c.getJSON(ctx, c.apiURL(fmt.Sprintf("/projects/%d/issues", projectID), query), &batch)+ if err != nil {+ return nil, false, err+ }+ documents := make([]gitlabBufferedDocument, 0, len(batch))+ doneByWindow := false+ pageOffset := 0+ for _, issue := range batch {+ if beforeOrAtWindowStart(issue.UpdatedAt, windowStart) {+ doneByWindow = true+ break+ }+ if afterWindowEnd(issue.UpdatedAt, windowEnd) {+ continue+ }+ doc := issue.toSourceDocument()+ pageOffset+++ documents = append(documents, gitlabBufferedDocument{+ document: doc,+ checkpoint: gitlabSyncCheckpoint(gitlabSyncCursor{Stage: gitlabStageIssues, Page: page, Offset: pageOffset, SourceID: doc.SourceID}, doc),+ offset: pageOffset,+ sourceID: doc.SourceID,+ })+ }+ done := doneByWindow || !gitlabHasNextPage(headers) || len(batch) == 0+ return documents, done, nil+}++// listTreePage returns one page of GitLab repository tree items.+func (c *GitlabConnector) listTreePage(ctx context.Context, projectID int, branch, path string, page, pageSize int) ([]gitlabTreeItem, bool, error) {+ query := url.Values{+ "per_page": {strconv.Itoa(pageSize)},+ "page": {strconv.Itoa(page)},+ "ref": {branch},+ }+ if path != "" {+ query.Set("path", path)+ }+ var batch []gitlabTreeItem+ headers, err := c.getJSON(ctx, c.apiURL(fmt.Sprintf("/projects/%d/repository/tree", projectID), query), &batch)+ if err != nil {+ return nil, false, err+ }+ done := !gitlabHasNextPage(headers) || len(batch) == 0+ return batch, done, nil+}++// fetchLastCommit returns the most recent commit for a file path.+func (c *GitlabConnector) fetchLastCommit(ctx context.Context, projectID int, branch, path string) (gitlabCommit, error) {+ query := url.Values{+ "ref_name": {branch},+ "path": {path},+ "per_page": {"1"},+ }+ var commits []gitlabCommit+ if _, err := c.getJSON(ctx, c.apiURL(fmt.Sprintf("/projects/%d/repository/commits", projectID), query), &commits); err != nil {+ return gitlabCommit{}, err+ }+ if len(commits) == 0 {+ return gitlabCommit{}, nil+ }+ return commits[0], nil+}++// fetchFileRaw downloads raw file content from GitLab.+func (c *GitlabConnector) fetchFileRaw(ctx context.Context, projectID int, branch, path string) ([]byte, error) {+ encoded := url.PathEscape(path)+ apiURL := c.apiURL(fmt.Sprintf("/projects/%d/repository/files/%s/raw", projectID, encoded), url.Values{"ref": {branch}})+ return c.getRaw(ctx, apiURL)+}++// getJSON fetches a GitLab API JSON response into out.+func (c *GitlabConnector) getJSON(ctx context.Context, apiURL string, out any) (http.Header, error) {+ if c.doJSON != nil {+ return c.doJSON(ctx, apiURL, out)+ }+ hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)+ if err != nil {+ return nil, err+ }+ client := utility.PinnedHTTPClient(hostname, resolvedIP, gitlabRequestTimeout)+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)+ if err != nil {+ return nil, err+ }+ req.Header.Set("Accept", "application/json")+ req.Header.Set("PRIVATE-TOKEN", c.token)+ resp, err := client.Do(req)+ if err != nil {+ return nil, fmt.Errorf("failed to fetch GitLab API: %w", err)+ }+ defer resp.Body.Close()+ if resp.StatusCode >= 400 {+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ return nil, fmt.Errorf("GitLab API returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))+ }+ if err = json.NewDecoder(resp.Body).Decode(out); err != nil {+ return nil, err+ }+ return resp.Header.Clone(), nil+}++// getRaw fetches raw bytes from a GitLab API URL.+func (c *GitlabConnector) getRaw(ctx context.Context, apiURL string) ([]byte, error) {+ if c.doRaw != nil {+ return c.doRaw(ctx, apiURL)+ }+ hostname, resolvedIP, err := utility.AssertURLSafe(apiURL)+ if err != nil {+ return nil, err+ }+ client := utility.PinnedHTTPClient(hostname, resolvedIP, gitlabRequestTimeout)+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil)+ if err != nil {+ return nil, err+ }+ req.Header.Set("PRIVATE-TOKEN", c.token)+ resp, err := client.Do(req)+ if err != nil {+ return nil, fmt.Errorf("failed to fetch GitLab raw file: %w", err)+ }+ defer resp.Body.Close()+ if resp.StatusCode >= 400 {+ body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))+ return nil, fmt.Errorf("GitLab API returned HTTP %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))+ }+ return io.ReadAll(resp.Body)+}++// apiURL builds a GitLab API URL.+func (c *GitlabConnector) apiURL(path string, query url.Values) string {+ path = strings.TrimLeft(path, "/")+ u := strings.TrimRight(c.baseURL, "/") + "/" + path+ if len(query) == 0 {+ return u+ }+ return u + "?" + query.Encode()+}++// fileURL builds the web URL for a repository file.+func (c *GitlabConnector) fileURL(branch, path string) string {+ return fmt.Sprintf("%s/%s/%s/-/blob/%s/%s", c.gitlabURL, c.projectOwner, c.projectName, branch, path)+}++// projectFullName returns "owner/name".+func (c *GitlabConnector) projectFullName() string {+ return c.projectOwner + "/" + c.projectName+}++// gitlabSyncSession streams GitLab documents for one fixed sync window.+type gitlabSyncSession struct {+ connector *GitlabConnector+ project gitlabProject+ batchSize int+ stage string+ page int+ treeQueue []string+ treePage int+ windowStart *time.Time+ windowEnd time.Time+ buffer []gitlabBufferedDocument+ resumeStage string+ resumePage int+ resumeOffset int+ resumeSourceID string+}++// NextBatch returns the next GitLab document batch.+func (s *gitlabSyncSession) NextBatch(ctx context.Context) (SyncBatch, error) {+ documents := make([]SourceDocument, 0, s.batchSize)+ var checkpoint *SyncCheckpoint+ if len(s.buffer) > 0 {+ n := s.batchSize+ if n > len(s.buffer) {+ n = len(s.buffer)+ }+ for _, buffered := range s.buffer[:n] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }+ s.buffer = s.buffer[n:]+ }++ for len(documents) < s.batchSize {+ batch, err := s.nextDocumentPage(ctx)+ if err != nil {+ return SyncBatch{}, err+ }+ if len(batch) == 0 {+ if s.stage == gitlabStageDone {+ if len(documents) == 0 {+ return SyncBatch{}, io.EOF+ }+ break+ }+ continue+ }+ remaining := s.batchSize - len(documents)+ if len(batch) > remaining {+ for _, buffered := range batch[:remaining] {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }+ s.buffer = append(s.buffer, batch[remaining:]...)+ break+ }+ for _, buffered := range batch {+ documents = append(documents, buffered.document)+ checkpoint = buffered.checkpoint+ }+ }+ return SyncBatch{Documents: documents, Checkpoint: checkpoint}, nil+}++// Close closes the GitLab sync session.+func (s *gitlabSyncSession) Close() error {+ return nil+}++// Fetch downloads a GitLab file body for a delayed source document.+func (s *gitlabSyncSession) Fetch(ctx context.Context, ref FetchReference) ([]byte, error) {+ var fetch gitlabFetchReference+ if err := json.Unmarshal([]byte(ref.Key), &fetch); err != nil {+ return nil, err+ }+ return s.connector.fetchFileRaw(ctx, fetch.ProjectID, fetch.Ref, fetch.FilePath)+}++const (+ gitlabStageCodeFiles = "code_files"+ gitlabStageMRs = "merge_requests"+ gitlabStageIssues = "issues"+ gitlabStageDone = "done"+)++// nextDocumentPage fetches one GitLab API page for sync.+func (s *gitlabSyncSession) nextDocumentPage(ctx context.Context) ([]gitlabBufferedDocument, error) {+ switch s.stage {+ case gitlabStageCodeFiles:+ if !s.connector.includeCodeFiles {+ s.advanceStage()+ return nil, nil+ }+ return s.nextCodeFilesPage(ctx)+ case gitlabStageMRs:+ if !s.connector.includeMRs {+ s.advanceStage()+ return nil, nil+ }+ docs, done, err := s.connector.listMergeRequestPage(ctx, s.project.ID, s.page, s.batchSize, s.windowStart, s.windowEnd)+ if err != nil {+ return nil, err+ }+ docs = s.filterResumedDocuments(gitlabStageMRs, s.page, docs)+ if done {+ s.advanceStage()+ } else {+ s.page+++ }+ return docs, nil+ case gitlabStageIssues:+ if !s.connector.includeIssues {+ s.advanceStage()+ return nil, nil+ }+ docs, done, err := s.connector.listIssuePage(ctx, s.project.ID, s.page, s.batchSize, s.windowStart, s.windowEnd)+ if err != nil {+ return nil, err+ }+ docs = s.filterResumedDocuments(gitlabStageIssues, s.page, docs)+ if done {+ s.advanceStage()+ } else {+ s.page+++ }+ return docs, nil+ default:+ s.stage = gitlabStageDone+ return nil, nil+ }+}++// nextCodeFilesPage fetches one tree page and converts blobs into documents.+// pendingPaths is captured before processing so resume re-discovers subtrees+// without duplicating queue entries.+func (s *gitlabSyncSession) nextCodeFilesPage(ctx context.Context) ([]gitlabBufferedDocument, error) {+ if len(s.treeQueue) == 0 {+ s.advanceStage()+ return nil, nil+ }+ currentPath := s.treeQueue[0]+ pendingPaths := append([]string(nil), s.treeQueue[1:]...)+ items, done, err := s.connector.listTreePage(ctx, s.project.ID, s.project.DefaultBranch, currentPath, s.treePage, s.batchSize)+ if err != nil {+ return nil, err+ }+ documents := make([]gitlabBufferedDocument, 0, len(items))+ pageOffset := 0+ for _, item := range items {+ if shouldExcludeGitlabPath(item.Path) {+ continue+ }+ if item.Type == "tree" {+ s.treeQueue = append(s.treeQueue, item.Path)+ continue+ }+ if item.Type != "blob" {+ continue+ }+ // The tree API does not expose a committed date, so this request still+ // supplies the update time used by window filtering.+ commit, err := s.connector.fetchLastCommit(ctx, s.project.ID, s.project.DefaultBranch, item.Path)+ if err != nil {+ return nil, err+ }+ updatedAt := time.Now().UTC()+ if !commit.CommittedDate.IsZero() {+ updatedAt = commit.CommittedDate.UTC()+ }+ if beforeOrAtWindowStart(updatedAt, s.windowStart) {+ continue+ }+ if afterWindowEnd(updatedAt, s.windowEnd) {+ continue+ }+ fileURL := s.connector.fileURL(s.project.DefaultBranch, item.Path)+ fetchRef := gitlabFetchReference{ProjectID: s.project.ID, FilePath: item.Path, Ref: s.project.DefaultBranch}+ fetchKey, _ := json.Marshal(fetchRef)+ doc := SourceDocument{+ SourceID: fileURL,+ SemanticIdentifier: item.Name,+ Extension: gitlabFileExt(item.Name),+ FetchRef: &FetchReference{Key: string(fetchKey), SizeHint: 0},+ UpdatedAt: updatedAt,+ Metadata: map[string]any{+ "type": "CodeFile",+ "path": item.Path,+ "ref": s.project.DefaultBranch,+ "project": s.connector.projectFullName(),+ "web_url": fileURL,+ },+ Fingerprint: gitlabCodeFileFingerprint(item, s.project.DefaultBranch),+ }+ pageOffset+++ cursor := gitlabSyncCursor{+ Stage: gitlabStageCodeFiles,+ Page: s.treePage,+ Offset: pageOffset,+ SourceID: doc.SourceID,+ TreePath: currentPath,+ PendingPaths: append([]string(nil), pendingPaths...),+ }+ documents = append(documents, gitlabBufferedDocument{+ document: doc,+ checkpoint: gitlabSyncCheckpoint(cursor, doc),+ offset: pageOffset,+ sourceID: doc.SourceID,+ })+ }+ documents = s.filterResumedDocuments(gitlabStageCodeFiles, s.treePage, documents)+ if done {+ s.treeQueue = s.treeQueue[1:]+ s.treePage = 1+ if len(s.treeQueue) == 0 {+ s.advanceStage()+ }+ } else {+ s.treePage+++ }+ return documents, nil+}++// advanceStage moves a sync session to the next GitLab stage.+func (s *gitlabSyncSession) advanceStage() {+ switch s.stage {+ case gitlabStageCodeFiles:+ s.stage = gitlabStageMRs+ case gitlabStageMRs:+ s.stage = gitlabStageIssues+ default:+ s.stage = gitlabStageDone+ }+ s.page = 1+ s.treeQueue = nil+ s.treePage = 1+ s.clearResume()+}++// applyResume advances a sync session to the last committed GitLab position.+func (s *gitlabSyncSession) applyResume(checkpoint *SyncCheckpoint) {+ if checkpoint == nil || checkpoint.Cursor == "" {+ return+ }+ var cursor gitlabSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ return+ }+ if cursor.Stage == "" || cursor.Page <= 0 {+ return+ }+ s.stage = cursor.Stage+ s.resumeStage = cursor.Stage+ s.resumePage = cursor.Page+ s.resumeOffset = cursor.Offset+ s.resumeSourceID = firstNonEmpty(cursor.SourceID, checkpoint.SourceID)+ if cursor.Stage == gitlabStageCodeFiles {+ s.treeQueue = append([]string{cursor.TreePath}, cursor.PendingPaths...)+ s.treePage = cursor.Page+ } else {+ s.page = cursor.Page+ }+}++// filterResumedDocuments drops documents through the committed checkpoint.+func (s *gitlabSyncSession) filterResumedDocuments(stage string, page int, candidates []gitlabBufferedDocument) []gitlabBufferedDocument {+ if s.resumeStage == "" || stage != s.resumeStage || page != s.resumePage {+ return candidates+ }+ if s.resumeSourceID != "" {+ for index, candidate := range candidates {+ if candidate.sourceID == s.resumeSourceID {+ s.clearResume()+ return candidates[index+1:]+ }+ }+ }+ if s.resumeOffset <= 0 {+ s.clearResume()+ return candidates+ }+ filtered := candidates[:0]+ for _, candidate := range candidates {+ if candidate.offset > s.resumeOffset {+ filtered = append(filtered, candidate)+ }+ }+ s.clearResume()+ return filtered+}++func (s *gitlabSyncSession) clearResume() {+ s.resumeStage = ""+ s.resumePage = 0+ s.resumeOffset = 0+ s.resumeSourceID = ""+}++// gitlabPruneSession streams a complete GitLab slim snapshot.+type gitlabPruneSession struct {+ connector *GitlabConnector+ project gitlabProject+ batchSize int+ stage string+ page int+ treeQueue []string+ treePage int+ buffer []SlimDocument+}++// NextBatch returns the next GitLab prune snapshot batch.+func (s *gitlabPruneSession) NextBatch(ctx context.Context) (PruneBatch, error) {+ documents := make([]SlimDocument, 0, s.batchSize)+ if len(s.buffer) > 0 {+ n := s.batchSize+ if n > len(s.buffer) {+ n = len(s.buffer)+ }+ documents = append(documents, s.buffer[:n]...)+ s.buffer = s.buffer[n:]+ }+ for len(documents) < s.batchSize {+ batch, err := s.nextSlimPage(ctx)+ if err != nil {+ return PruneBatch{}, err+ }+ if len(batch) == 0 {+ if s.stage == gitlabStageDone {+ if len(documents) == 0 {+ return PruneBatch{}, io.EOF+ }+ break+ }+ continue+ }+ remaining := s.batchSize - len(documents)+ if len(batch) > remaining {+ documents = append(documents, batch[:remaining]...)+ s.buffer = append(s.buffer, batch[remaining:]...)+ break+ }+ documents = append(documents, batch...)+ }+ return PruneBatch{Documents: documents}, nil+}++// Close closes the GitLab prune session.+func (s *gitlabPruneSession) Close() error {+ return nil+}++// nextSlimPage fetches one GitLab API page for prune.+func (s *gitlabPruneSession) nextSlimPage(ctx context.Context) ([]SlimDocument, error) {+ switch s.stage {+ case gitlabStageCodeFiles:+ if !s.connector.includeCodeFiles {+ s.advanceStage()+ return nil, nil+ }+ return s.nextCodeFilesSlimPage(ctx)+ case gitlabStageMRs:+ if !s.connector.includeMRs {+ s.advanceStage()+ return nil, nil+ }+ return s.nextMRSlimPage(ctx)+ case gitlabStageIssues:+ if !s.connector.includeIssues {+ s.advanceStage()+ return nil, nil+ }+ return s.nextIssueSlimPage(ctx)+ default:+ s.stage = gitlabStageDone+ return nil, nil+ }+}++// nextCodeFilesSlimPage fetches one tree page for prune.+func (s *gitlabPruneSession) nextCodeFilesSlimPage(ctx context.Context) ([]SlimDocument, error) {+ if len(s.treeQueue) == 0 {+ s.advanceStage()+ return nil, nil+ }+ currentPath := s.treeQueue[0]+ items, done, err := s.connector.listTreePage(ctx, s.project.ID, s.project.DefaultBranch, currentPath, s.treePage, s.batchSize)+ if err != nil {+ return nil, err+ }+ documents := make([]SlimDocument, 0, len(items))+ for _, item := range items {+ if shouldExcludeGitlabPath(item.Path) {+ continue+ }+ if item.Type == "tree" {+ s.treeQueue = append(s.treeQueue, item.Path)+ continue+ }+ if item.Type != "blob" {+ continue+ }+ documents = append(documents, SlimDocument{SourceID: s.connector.fileURL(s.project.DefaultBranch, item.Path)})+ }+ if done {+ s.treeQueue = s.treeQueue[1:]+ s.treePage = 1+ if len(s.treeQueue) == 0 {+ s.advanceStage()+ }+ } else {+ s.treePage+++ }+ return documents, nil+}++// nextMRSlimPage fetches one MR page for prune.+func (s *gitlabPruneSession) nextMRSlimPage(ctx context.Context) ([]SlimDocument, error) {+ query := gitlabListQuery(s.page, s.batchSize)+ var batch []gitlabMergeRequest+ headers, err := s.connector.getJSON(ctx, s.connector.apiURL(fmt.Sprintf("/projects/%d/merge_requests", s.project.ID), query), &batch)+ if err != nil {+ return nil, err+ }+ documents := make([]SlimDocument, 0, len(batch))+ for _, mr := range batch {+ documents = append(documents, SlimDocument{SourceID: mr.WebURL})+ }+ if !gitlabHasNextPage(headers) || len(batch) == 0 {+ s.advanceStage()+ } else {+ s.page+++ }+ return documents, nil+}++// nextIssueSlimPage fetches one issue page for prune.+func (s *gitlabPruneSession) nextIssueSlimPage(ctx context.Context) ([]SlimDocument, error) {+ query := gitlabListQuery(s.page, s.batchSize)+ var batch []gitlabIssue+ headers, err := s.connector.getJSON(ctx, s.connector.apiURL(fmt.Sprintf("/projects/%d/issues", s.project.ID), query), &batch)+ if err != nil {+ return nil, err+ }+ documents := make([]SlimDocument, 0, len(batch))+ for _, issue := range batch {+ documents = append(documents, SlimDocument{SourceID: issue.WebURL})+ }+ if !gitlabHasNextPage(headers) || len(batch) == 0 {+ s.advanceStage()+ } else {+ s.page+++ }+ return documents, nil+}++// advanceStage moves a prune session to the next GitLab stage.+func (s *gitlabPruneSession) advanceStage() {+ switch s.stage {+ case gitlabStageCodeFiles:+ s.stage = gitlabStageMRs+ case gitlabStageMRs:+ s.stage = gitlabStageIssues+ default:+ s.stage = gitlabStageDone+ }+ s.page = 1+ s.treeQueue = nil+ s.treePage = 1+}++type gitlabProject struct {+ ID int `json:"id"`+ DefaultBranch string `json:"default_branch"`+ WebURL string `json:"web_url"`+}++type gitlabSyncCursor struct {+ Stage string `json:"stage"`+ Page int `json:"page"`+ Offset int `json:"offset"`+ SourceID string `json:"source_id"`+ TreePath string `json:"tree_path,omitempty"`+ PendingPaths []string `json:"pending_paths,omitempty"`+}++type gitlabBufferedDocument struct {+ document SourceDocument+ checkpoint *SyncCheckpoint+ offset int+ sourceID string+}++type gitlabFetchReference struct {+ ProjectID int `json:"project_id"`+ FilePath string `json:"file_path"`+ Ref string `json:"ref"`+}++func gitlabSyncCheckpoint(cursor gitlabSyncCursor, doc SourceDocument) *SyncCheckpoint {+ data, err := json.Marshal(cursor)+ if err != nil {+ return nil+ }+ updatedAt := doc.UpdatedAt+ return &SyncCheckpoint{+ Cursor: string(data),+ SourceID: doc.SourceID,+ UpdatedAt: &updatedAt,+ }+}++type gitlabMergeRequest struct {+ IID int `json:"iid"`+ Title string `json:"title"`+ Description string `json:"description"`+ State string `json:"state"`+ WebURL string `json:"web_url"`+ UpdatedAt time.Time `json:"updated_at"`+ Author *gitlabUser `json:"author"`+}++// toSourceDocument converts a merge request into the syncer model.+func (m gitlabMergeRequest) toSourceDocument() SourceDocument {+ body := []byte(m.Description)+ author := m.Author.metadata()+ return SourceDocument{+ SourceID: m.WebURL,+ SemanticIdentifier: m.Title,+ Extension: ".md",+ Blob: body,+ UpdatedAt: m.UpdatedAt.UTC(),+ SizeBytes: int64(len(body)),+ Metadata: map[string]any{+ "state": m.State,+ "type": "MergeRequest",+ "web_url": m.WebURL,+ "author": author,+ },+ Fingerprint: stableFingerprint(map[string]any{+ "type": "MergeRequest",+ "url": m.WebURL,+ "title": m.Title,+ "description": m.Description,+ "state": m.State,+ "updated_at": m.UpdatedAt.UTC(),+ "author": author,+ }),+ }+}++type gitlabIssue struct {+ IID int `json:"iid"`+ Title string `json:"title"`+ Description string `json:"description"`+ State string `json:"state"`+ WebURL string `json:"web_url"`+ UpdatedAt time.Time `json:"updated_at"`+ Author *gitlabUser `json:"author"`+ Type string `json:"type"`+}++// toSourceDocument converts an issue into the syncer model.+func (i gitlabIssue) toSourceDocument() SourceDocument {+ body := []byte(i.Description)+ author := i.Author.metadata()+ issueType := i.Type+ if issueType == "" {+ issueType = "Issue"+ }+ return SourceDocument{+ SourceID: i.WebURL,+ SemanticIdentifier: i.Title,+ Extension: ".md",+ Blob: body,+ UpdatedAt: i.UpdatedAt.UTC(),+ SizeBytes: int64(len(body)),+ Metadata: map[string]any{+ "state": i.State,+ "type": issueType,+ "web_url": i.WebURL,+ "author": author,+ },+ Fingerprint: stableFingerprint(map[string]any{+ "type": issueType,+ "url": i.WebURL,+ "title": i.Title,+ "description": i.Description,+ "state": i.State,+ "updated_at": i.UpdatedAt.UTC(),+ "author": author,+ }),+ }+}++type gitlabTreeItem struct {+ ID string `json:"id"`+ Name string `json:"name"`+ Type string `json:"type"`+ Path string `json:"path"`+ Mode string `json:"mode"`+}++func gitlabCodeFileFingerprint(item gitlabTreeItem, ref string) string {+ return stableFingerprint(map[string]any{+ "type": "CodeFile",+ "path": item.Path,+ "ref": ref,+ "id": item.ID,+ })+}++type gitlabCommit struct {+ ID string `json:"id"`+ CommittedDate time.Time `json:"committed_date"`+}++type gitlabUser struct {+ Name string `json:"name"`+ Username string `json:"username"`+}++// metadata returns user metadata.+func (u *gitlabUser) metadata() map[string]string {+ if u == nil {+ return nil+ }+ out := map[string]string{}+ if u.Name != "" {+ out["name"] = u.Name+ }+ if u.Username != "" {+ out["username"] = u.Username+ }+ return out+}++// gitlabListQuery builds standard GitLab list query parameters.+func gitlabListQuery(page, pageSize int) url.Values {+ if pageSize <= 0 {+ pageSize = defaultGitlabBatchSize+ }+ return url.Values{+ "state": {"all"},+ "per_page": {strconv.Itoa(pageSize)},+ "page": {strconv.Itoa(page)},+ }+}++// gitlabHasNextPage reports whether a GitLab response has a next page.+func gitlabHasNextPage(headers http.Header) bool {+ nextPage := strings.TrimSpace(headers.Get("X-Next-Page"))+ if nextPage != "" && nextPage != "0" {+ return true+ }+ return strings.Contains(headers.Get("Link"), `rel="next"`)+}++// shouldExcludeGitlabPath reports whether a path matches an exclude pattern.+func shouldExcludeGitlabPath(path string) bool {+ for _, pattern := range gitlabExcludePatterns {+ pattern = strings.TrimSuffix(pattern, "/")+ if path == pattern || strings.HasPrefix(path, pattern+"/") {+ return true+ }+ }+ return false+}++// gitlabFileExt returns the lowercased file extension.+func gitlabFileExt(name string) string {+ ext := filepath.Ext(name)+ if ext == "" {+ return ""+ }+ return strings.ToLower(ext)+}++// gitlabProjectPath URL-encodes "owner/name" for the GitLab projects API.+func gitlabProjectPath(owner, name string) string {+ return url.QueryEscape(owner + "/" + name)+}diff --git a/internal/syncer/connector/gitlab_test.go b/internal/syncer/connector/gitlab_test.gonew file mode 100644index 0000000000..c2863ca49d--- /dev/null+++ b/internal/syncer/connector/gitlab_test.go@@ -0,0 +1,458 @@+package connector++import (+ "context"+ "encoding/json"+ "errors"+ "io"+ "net/http"+ "net/url"+ "strings"+ "testing"+)++// TestGitlabConnectorOpenSyncUsesWindowAndFingerprint verifies incremental sync emits only updated docs with fingerprints.+func TestGitlabConnectorOpenSyncUsesWindowAndFingerprint(t *testing.T) {+ connector, err := NewGitlabConnector(map[string]any{+ "project_owner": "owner",+ "project_name": "repo",+ "include_mrs": true,+ "include_issues": true,+ "include_code_files": false,+ "batch_size": 10,+ "credentials": map[string]any{"gitlab_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitlabConnector failed: %v", err)+ }+ connector.baseURL = "https://gitlab.com/api/v4"+ connector.doJSON = gitlabFixtureDoJSON(t)++ start := mustTime(t, "2026-01-02T12:00:00Z")+ end := mustTime(t, "2026-01-04T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{WindowStart: &start, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 1 {+ t.Fatalf("documents len = %d, want 1", len(batch.Documents))+ }+ doc := batch.Documents[0]+ if doc.SourceID != "https://gitlab.com/owner/repo/-/merge_requests/7" {+ t.Fatalf("source id = %q", doc.SourceID)+ }+ if doc.Fingerprint == "" {+ t.Fatalf("fingerprint is empty")+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++// TestGitlabFingerprintStable verifies GitLab fingerprints are stable and content-sensitive.+func TestGitlabFingerprintStable(t *testing.T) {+ updatedAt := mustTime(t, "2026-01-03T00:00:00Z")+ mr := gitlabMergeRequest{+ IID: 7,+ Title: "Add syncer",+ Description: "MR body",+ State: "opened",+ WebURL: "https://gitlab.com/owner/repo/-/merge_requests/7",+ UpdatedAt: updatedAt,+ Author: &gitlabUser{Name: "Alice", Username: "alice"},+ }+ fp1 := mr.toSourceDocument().Fingerprint+ fp2 := mr.toSourceDocument().Fingerprint+ if fp1 == "" || fp1 != fp2 {+ t.Fatalf("fingerprint unstable: %q %q", fp1, fp2)+ }++ changed := mr+ changed.Title = "Add syncer v2"+ if got := changed.toSourceDocument().Fingerprint; got == fp1 {+ t.Fatalf("fingerprint did not change after title update")+ }+}++func TestGitlabCodeFileFingerprintUsesTreeItemID(t *testing.T) {+ item := gitlabTreeItem{ID: "abc", Name: "main.go", Type: "blob", Path: "main.go"}+ base := gitlabCodeFileFingerprint(item, "main")+ if base == "" {+ t.Fatalf("fingerprint is empty")+ }+ if got := gitlabCodeFileFingerprint(item, "main"); got != base {+ t.Fatalf("fingerprint unstable: %q %q", base, got)+ }++ changed := item+ changed.ID = "def"+ if got := gitlabCodeFileFingerprint(changed, "main"); got == base {+ t.Fatalf("fingerprint did not change after tree item ID update")+ }+}++// TestGitlabConnectorOpenPrune verifies PRUNE returns correct web_url IDs.+func TestGitlabConnectorOpenPrune(t *testing.T) {+ connector, err := NewGitlabConnector(map[string]any{+ "project_owner": "owner",+ "project_name": "repo",+ "include_mrs": true,+ "include_issues": true,+ "include_code_files": false,+ "batch_size": 10,+ "credentials": map[string]any{"gitlab_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitlabConnector failed: %v", err)+ }+ connector.baseURL = "https://gitlab.com/api/v4"+ connector.doJSON = gitlabFixtureDoJSON(t)++ session, err := connector.OpenPrune(context.Background(), PruneRequest{})+ if err != nil {+ t.Fatalf("OpenPrune failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ got := []string{}+ for _, doc := range batch.Documents {+ got = append(got, doc.SourceID)+ }+ want := []string{+ "https://gitlab.com/owner/repo/-/merge_requests/7",+ "https://gitlab.com/owner/repo/-/issues/3",+ }+ if len(got) != len(want) {+ t.Fatalf("ids len = %d, want %d: %v", len(got), len(want), got)+ }+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("ids[%d] = %q, want %q", i, got[i], want[i])+ }+ }+}++// TestGitlabConnectorOpenSyncResumesAfterCheckpoint verifies retry skips committed GitLab documents.+func TestGitlabConnectorOpenSyncResumesAfterCheckpoint(t *testing.T) {+ connector, err := NewGitlabConnector(map[string]any{+ "project_owner": "owner",+ "project_name": "repo",+ "include_mrs": true,+ "include_issues": true,+ "include_code_files": false,+ "batch_size": 1,+ "credentials": map[string]any{"gitlab_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitlabConnector failed: %v", err)+ }+ connector.baseURL = "https://gitlab.com/api/v4"+ connector.doJSON = gitlabFixtureDoJSON(t)++ end := mustTime(t, "2026-01-04T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch first failed: %v", err)+ }+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "https://gitlab.com/owner/repo/-/merge_requests/7" {+ t.Fatalf("first documents = %+v, want MR 7", first.Documents)+ }+ if first.Checkpoint == nil || first.Checkpoint.SourceID != "https://gitlab.com/owner/repo/-/merge_requests/7" {+ t.Fatalf("first checkpoint = %+v, want MR 7", first.Checkpoint)+ }++ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end, Resume: first.Checkpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "https://gitlab.com/owner/repo/-/issues/3" {+ t.Fatalf("resume documents = %+v, want issue 3", second.Documents)+ }+ if second.Checkpoint == nil || second.Checkpoint.SourceID != "https://gitlab.com/owner/repo/-/issues/3" {+ t.Fatalf("resume checkpoint = %+v, want issue 3", second.Checkpoint)+ }+ if _, err = resumed.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("resume EOF = %v", err)+ }+}++// TestGitlabConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument verifies offset fallback excludes the committed offset.+func TestGitlabConnectorOpenSyncResumeOffsetFallbackSkipsCommittedDocument(t *testing.T) {+ connector, err := NewGitlabConnector(map[string]any{+ "project_owner": "owner",+ "project_name": "repo",+ "include_mrs": true,+ "include_issues": true,+ "include_code_files": false,+ "batch_size": 1,+ "credentials": map[string]any{"gitlab_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitlabConnector failed: %v", err)+ }+ connector.baseURL = "https://gitlab.com/api/v4"+ connector.doJSON = gitlabFixtureDoJSON(t)++ end := mustTime(t, "2026-01-04T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ first, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch first failed: %v", err)+ }+ if len(first.Documents) != 1 || first.Documents[0].SourceID != "https://gitlab.com/owner/repo/-/merge_requests/7" {+ t.Fatalf("first documents = %+v, want MR 7", first.Documents)+ }+ if first.Checkpoint == nil {+ t.Fatalf("first checkpoint is nil")+ }++ resumeCheckpoint := cloneGitlabCheckpointWithMissingSourceID(t, first.Checkpoint)+ resumed, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end, Resume: resumeCheckpoint})+ if err != nil {+ t.Fatalf("resume OpenSync failed: %v", err)+ }+ second, err := resumed.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("resume NextBatch failed: %v", err)+ }+ if len(second.Documents) != 1 || second.Documents[0].SourceID != "https://gitlab.com/owner/repo/-/issues/3" {+ t.Fatalf("resume documents = %+v, want issue 3", second.Documents)+ }+ if second.Documents[0].SourceID == first.Documents[0].SourceID {+ t.Fatalf("committed document was redelivered: %s", second.Documents[0].SourceID)+ }+}++// TestGitlabConnectorOpenSyncCodeFiles verifies BFS traversal of code files with lazy Fetch.+func TestGitlabConnectorOpenSyncCodeFiles(t *testing.T) {+ connector, err := NewGitlabConnector(map[string]any{+ "project_owner": "owner",+ "project_name": "repo",+ "include_mrs": false,+ "include_issues": false,+ "include_code_files": true,+ "batch_size": 10,+ "credentials": map[string]any{"gitlab_access_token": "token"},+ })+ if err != nil {+ t.Fatalf("NewGitlabConnector failed: %v", err)+ }+ connector.baseURL = "https://gitlab.com/api/v4"+ connector.doJSON = gitlabFixtureDoJSON(t)+ connector.doRaw = gitlabFixtureDoRaw(t)++ end := mustTime(t, "2026-01-04T00:00:00Z")+ session, err := connector.OpenSync(context.Background(), SyncRequest{FromBeginning: true, WindowEnd: end})+ if err != nil {+ t.Fatalf("OpenSync failed: %v", err)+ }+ batch, err := session.NextBatch(context.Background())+ if err != nil {+ t.Fatalf("NextBatch failed: %v", err)+ }+ if len(batch.Documents) != 2 {+ t.Fatalf("documents len = %d, want 2 (main.go + src/helper.go)", len(batch.Documents))+ }+ if batch.Documents[0].SourceID != "https://gitlab.com/owner/repo/-/blob/main/main.go" {+ t.Fatalf("doc[0] source id = %q", batch.Documents[0].SourceID)+ }+ if batch.Documents[1].SourceID != "https://gitlab.com/owner/repo/-/blob/main/src/helper.go" {+ t.Fatalf("doc[1] source id = %q", batch.Documents[1].SourceID)+ }+ if batch.Documents[0].FetchRef == nil {+ t.Fatalf("doc[0] has no FetchRef")+ }+ fetcher, ok := session.(Fetcher)+ if !ok {+ t.Fatalf("session does not implement Fetcher")+ }+ blob, err := fetcher.Fetch(context.Background(), *batch.Documents[0].FetchRef)+ if err != nil {+ t.Fatalf("Fetch failed: %v", err)+ }+ if string(blob) != "package main\n" {+ t.Fatalf("blob = %q, want %q", string(blob), "package main\n")+ }+ if _, err = session.NextBatch(context.Background()); !errors.Is(err, io.EOF) {+ t.Fatalf("NextBatch EOF = %v", err)+ }+}++// TestGitlabFetchFileRawEscapesFilePath verifies raw file paths use path-safe %20/%2F encoding.+func TestGitlabFetchFileRawEscapesFilePath(t *testing.T) {+ connector := &GitlabConnector{baseURL: "https://gitlab.com/api/v4"}+ connector.doRaw = func(ctx context.Context, apiURL string) ([]byte, error) {+ parsed, err := url.Parse(apiURL)+ if err != nil {+ t.Fatalf("parse api url: %v", err)+ }+ if got := parsed.EscapedPath(); got != "/api/v4/projects/1/repository/files/src%20dir%2Fhelper%20one.go/raw" {+ t.Fatalf("escaped path = %q", got)+ }+ if got := parsed.Query().Get("ref"); got != "main" {+ t.Fatalf("ref = %q, want %q", got, "main")+ }+ return []byte("ok"), nil+ }+ body, err := connector.fetchFileRaw(context.Background(), 1, "main", "src dir/helper one.go")+ if err != nil {+ t.Fatalf("fetchFileRaw failed: %v", err)+ }+ if string(body) != "ok" {+ t.Fatalf("body = %q, want %q", body, "ok")+ }+}++// TestGitlabConnectorValidate verifies missing config returns clear errors.+func TestGitlabConnectorValidate(t *testing.T) {+ cases := []struct {+ name string+ config map[string]any+ want string+ }{+ {"missing owner", map[string]any{+ "project_name": "repo",+ "credentials": map[string]any{"gitlab_access_token": "token"},+ }, "project_owner"},+ {"missing token", map[string]any{+ "project_owner": "owner",+ "project_name": "repo",+ "credentials": map[string]any{},+ }, "gitlab_access_token"},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ connector, err := NewGitlabConnector(tc.config)+ if err != nil {+ t.Fatalf("NewGitlabConnector failed: %v", err)+ }+ err = connector.Validate(context.Background())+ if err == nil || !strings.Contains(err.Error(), tc.want) {+ t.Fatalf("err = %v, want contains %q", err, tc.want)+ }+ })+ }+}++func cloneGitlabCheckpointWithMissingSourceID(t *testing.T, checkpoint *SyncCheckpoint) *SyncCheckpoint {+ t.Helper()+ var cursor gitlabSyncCursor+ if err := json.Unmarshal([]byte(checkpoint.Cursor), &cursor); err != nil {+ t.Fatalf("decode checkpoint cursor: %v", err)+ }+ cursor.SourceID = ""+ data, err := json.Marshal(cursor)+ if err != nil {+ t.Fatalf("encode checkpoint cursor: %v", err)+ }+ clone := *checkpoint+ clone.Cursor = string(data)+ clone.SourceID = ""+ return &clone+}++// gitlabFixtureDoJSON returns a fixture GitLab JSON transport.+func gitlabFixtureDoJSON(t *testing.T) func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ t.Helper()+ return func(ctx context.Context, apiURL string, out any) (http.Header, error) {+ parsed, err := url.Parse(apiURL)+ if err != nil {+ t.Fatalf("parse api url: %v", err)+ }+ path := parsed.EscapedPath()+ query := parsed.Query()++ var body string+ switch {+ case strings.HasSuffix(path, "/projects/owner%2Frepo"):+ body = `{"id":1,"default_branch":"main","web_url":"https://gitlab.com/owner/repo"}`+ case strings.HasSuffix(path, "/projects/1/merge_requests"):+ body = `[{+ "iid":7,+ "title":"Add syncer",+ "description":"MR body",+ "state":"opened",+ "web_url":"https://gitlab.com/owner/repo/-/merge_requests/7",+ "updated_at":"2026-01-03T00:00:00.000Z",+ "author":{"name":"Alice","username":"alice"}+ }]`+ case strings.HasSuffix(path, "/projects/1/issues"):+ body = `[{+ "iid":3,+ "title":"Prune bug",+ "description":"Issue body",+ "state":"opened",+ "web_url":"https://gitlab.com/owner/repo/-/issues/3",+ "updated_at":"2026-01-02T00:00:00.000Z",+ "author":{"name":"Bob","username":"bob"},+ "type":"ISSUE"+ }]`+ case strings.HasSuffix(path, "/projects/1/repository/tree"):+ treePath := query.Get("path")+ if treePath == "" {+ body = `[+ {"id":"abc","name":"main.go","type":"blob","path":"main.go","mode":"100644"},+ {"id":"def","name":"src","type":"tree","path":"src","mode":"040000"}+ ]`+ } else if treePath == "src" {+ body = `[+ {"id":"ghi","name":"helper.go","type":"blob","path":"src/helper.go","mode":"100644"}+ ]`+ } else {+ t.Fatalf("unexpected tree path %s", treePath)+ }+ case strings.HasSuffix(path, "/projects/1/repository/commits"):+ commitPath := query.Get("path")+ if commitPath == "main.go" {+ body = `[{"id":"commit1","committed_date":"2026-01-03T00:00:00.000Z"}]`+ } else if commitPath == "src/helper.go" {+ body = `[{"id":"commit2","committed_date":"2026-01-02T00:00:00.000Z"}]`+ } else {+ t.Fatalf("unexpected commit path %s", commitPath)+ }+ default:+ t.Fatalf("unexpected api path %s", path)+ }+ if err = json.Unmarshal([]byte(body), out); err != nil {+ t.Fatalf("decode fixture: %v", err)+ }+ return http.Header{}, nil+ }+}++// gitlabFixtureDoRaw returns fixture raw file content.+func gitlabFixtureDoRaw(t *testing.T) func(ctx context.Context, apiURL string) ([]byte, error) {+ t.Helper()+ return func(ctx context.Context, apiURL string) ([]byte, error) {+ parsed, err := url.Parse(apiURL)+ if err != nil {+ t.Fatalf("parse api url: %v", err)+ }+ path := parsed.EscapedPath()+ if strings.HasSuffix(path, "/files/main.go/raw") {+ return []byte("package main\n"), nil+ }+ if strings.HasSuffix(path, "/files/src%2Fhelper.go/raw") {+ return []byte("package src\n"), nil+ }+ t.Fatalf("unexpected raw path %s", path)+ return nil, nil+ }+}