Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(datasource): fail closed on GitLab compare errors

hahahahahayesyeseys11edd16819b2
diff --git a/internal/datasource/connector/gitlab/connector.go b/internal/datasource/connector/gitlab/connector.go
index b82419541..0a71e7a99 100644
--- a/internal/datasource/connector/gitlab/connector.go
+++ b/internal/datasource/connector/gitlab/connector.go
@@ -178,39 +178,26 @@ func (c *Connector) FetchIncremental(ctx context.Context, ds *types.DataSourceCo
out = append(out, item)
}
} else if previous != head {
- diff, err := c.client.compare(ctx, s.ProjectID, previous, head)
- if err != nil || diff.CompareTimeout {
- // A compare may be unavailable after history rewrites or truncated by
- // the server. Re-enumerate the configured scope to preserve updates.
- files, listErr := c.files(ctx, s.ProjectID, ref, s.Paths)
- if listErr != nil {
- return nil, nil, fmt.Errorf("gitlab list files %s: %w", s.ProjectID, listErr)
- }
- for _, f := range files {
- item, itemErr := c.item(ctx, p, ref, f)
- if itemErr != nil {
- return nil, nil, itemErr
- }
- out = append(out, item)
- }
- } else {
- for _, d := range diff.Diffs {
- if d.DeletedFile {
- if c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
- out = append(out, c.deleted(p, ref, d.OldPath))
- }
- continue
- }
- if d.RenamedFile && c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
+ diff, err := c.compareChanges(ctx, s.ProjectID, previous, head)
+ if err != nil {
+ return nil, nil, err
+ }
+ for _, d := range diff.Diffs {
+ if d.DeletedFile {
+ if c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
out = append(out, c.deleted(p, ref, d.OldPath))
}
- if c.inScope(d.NewPath, s.Paths) && isSupportedFile(d.NewPath) {
- item, err := c.item(ctx, p, ref, d.NewPath)
- if err != nil {
- return nil, nil, err
- }
- out = append(out, item)
+ continue
+ }
+ if d.RenamedFile && c.inScope(d.OldPath, s.Paths) && isSupportedFile(d.OldPath) {
+ out = append(out, c.deleted(p, ref, d.OldPath))
+ }
+ if c.inScope(d.NewPath, s.Paths) && isSupportedFile(d.NewPath) {
+ item, err := c.item(ctx, p, ref, d.NewPath)
+ if err != nil {
+ return nil, nil, err
}
+ out = append(out, item)
}
}
}
@@ -294,11 +281,9 @@ func gitLabCursor(value cursor) *types.SyncCursor {
func (c *Connector) streamChanges(
ctx context.Context, project *project, ref, from, to string, roots []string, h datasource.StreamHandler,
) error {
- diff, err := c.client.compare(ctx, fmt.Sprint(project.ID), from, to)
- if err != nil || diff.CompareTimeout {
- // A compare can be unavailable after history rewrites or be truncated by
- // GitLab. Re-enumerating the configured scope preserves file updates.
- return c.streamFiles(ctx, project, ref, roots, h)
+ diff, err := c.compareChanges(ctx, fmt.Sprint(project.ID), from, to)
+ if err != nil {
+ return err
}
for _, change := range diff.Diffs {
if change.DeletedFile {
@@ -327,6 +312,17 @@ func (c *Connector) streamChanges(
return nil
}
+func (c *Connector) compareChanges(ctx context.Context, projectID, from, to string) (*comparison, error) {
+ diff, err := c.client.compare(ctx, projectID, from, to)
+ if err != nil {
+ return nil, fmt.Errorf("gitlab compare project %s: %w", projectID, err)
+ }
+ if diff.CompareTimeout {
+ return nil, fmt.Errorf("gitlab compare project %s timed out", projectID)
+ }
+ return diff, nil
+}
+
func (c *Connector) streamFiles(
ctx context.Context, project *project, ref string, roots []string, h datasource.StreamHandler,
) error {
diff --git a/internal/datasource/connector/gitlab/connector_recursion_test.go b/internal/datasource/connector/gitlab/connector_recursion_test.go
index f3d82bac3..5025ffc7f 100644
--- a/internal/datasource/connector/gitlab/connector_recursion_test.go
+++ b/internal/datasource/connector/gitlab/connector_recursion_test.go
@@ -19,7 +19,7 @@ import (
// on the second page. Browsing remains shallow; fetching must visit every level.
func TestConnectorFetchesNestedDirectories(t *testing.T) {
for _, mode := range []string{
- "all", "incremental", "stream", "incremental compare fallback", "stream compare fallback",
+ "all", "incremental", "stream",
} {
t.Run(mode, func(t *testing.T) {
allowLocalGitLabServer(t)
@@ -107,3 +107,149 @@ func TestConnectorFetchesNestedDirectories(t *testing.T) {
})
}
}
+
+func TestConnectorCompareFailuresDoNotFallBackToFullSync(t *testing.T) {
+ for _, failure := range []struct {
+ name string
+ statusCode int
+ body string
+ }{
+ {name: "timeout response", statusCode: http.StatusOK, body: `{"compare_timeout":true}`},
+ {name: "API error", statusCode: http.StatusInternalServerError, body: `{"message":"compare failed"}`},
+ } {
+ for _, mode := range []string{"incremental", "stream"} {
+ t.Run(failure.name+" "+mode, func(t *testing.T) {
+ allowLocalGitLabServer(t)
+ downstreamRequests := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api/v4/projects/1":
+ _, _ = w.Write([]byte(`{
+ "id":1,"name":"docs","path_with_namespace":"group/docs","default_branch":"main"
+ }`))
+ case "/api/v4/projects/1/repository/commits/main":
+ _, _ = w.Write([]byte(`{"id":"head"}`))
+ case "/api/v4/projects/1/repository/compare":
+ w.WriteHeader(failure.statusCode)
+ _, _ = w.Write([]byte(failure.body))
+ case "/api/v4/projects/1/repository/tree":
+ downstreamRequests++
+ _, _ = w.Write([]byte(`[{"name":"README.md","path":"README.md","type":"blob"}]`))
+ default:
+ if strings.Contains(r.URL.EscapedPath(), "/repository/files/") {
+ downstreamRequests++
+ _, _ = w.Write([]byte("# unchanged"))
+ return
+ }
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ cfg := &types.DataSourceConfig{
+ Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"},
+ Settings: map[string]interface{}{"projects": []interface{}{
+ map[string]interface{}{"project_id": "1", "paths": []interface{}{}},
+ }},
+ }
+ old := gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}})
+ connector := NewConnector()
+
+ switch mode {
+ case "incremental":
+ items, next, err := connector.FetchIncremental(context.Background(), cfg, old)
+ require.ErrorContains(t, err, "gitlab compare project 1")
+ require.Empty(t, items)
+ require.Nil(t, next)
+ case "stream":
+ handler := &gitLabStreamRecorder{}
+ next, err := connector.FetchStream(context.Background(), cfg, old, handler)
+ require.ErrorContains(t, err, "gitlab compare project 1")
+ require.Nil(t, next)
+ require.Empty(t, handler.items)
+ require.Empty(t, handler.checkpoints)
+ }
+ require.Zero(t, downstreamRequests, "compare failures must not trigger a full repository walk")
+ })
+ }
+ }
+}
+
+func TestConnectorSuccessfulCompareStillEmitsOnlyChangedFiles(t *testing.T) {
+ for _, mode := range []string{"incremental", "stream"} {
+ t.Run(mode, func(t *testing.T) {
+ allowLocalGitLabServer(t)
+ var compareQuery string
+ treeRequests := 0
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api/v4/projects/1":
+ _, _ = w.Write([]byte(`{
+ "id":1,"name":"docs","path_with_namespace":"group/docs",
+ "web_url":"https://gitlab.test/group/docs","default_branch":"main"
+ }`))
+ case "/api/v4/projects/1/repository/commits/main":
+ _, _ = w.Write([]byte(`{"id":"head"}`))
+ case "/api/v4/projects/1/repository/compare":
+ compareQuery = r.URL.RawQuery
+ _, _ = w.Write([]byte(`{"diffs":[
+ {"old_path":"removed.md","new_path":"removed.md","deleted_file":true},
+ {"old_path":"old.md","new_path":"renamed.md","renamed_file":true},
+ {"old_path":"guide.md","new_path":"guide.md"},
+ {"old_path":"server.go","new_path":"server.go"}
+ ]}`))
+ case "/api/v4/projects/1/repository/tree":
+ treeRequests++
+ http.Error(w, "unexpected full repository walk", http.StatusInternalServerError)
+ default:
+ if strings.Contains(r.URL.EscapedPath(), "/repository/files/") {
+ _, _ = w.Write([]byte("# changed"))
+ return
+ }
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+
+ cfg := &types.DataSourceConfig{
+ Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"},
+ Settings: map[string]interface{}{"projects": []interface{}{
+ map[string]interface{}{"project_id": "1", "paths": []interface{}{}},
+ }},
+ }
+ old := gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}})
+ connector := NewConnector()
+ var items []types.FetchedItem
+ var next *types.SyncCursor
+
+ switch mode {
+ case "incremental":
+ var err error
+ items, next, err = connector.FetchIncremental(context.Background(), cfg, old)
+ require.NoError(t, err)
+ case "stream":
+ handler := &gitLabStreamRecorder{}
+ var err error
+ next, err = connector.FetchStream(context.Background(), cfg, old, handler)
+ require.NoError(t, err)
+ items = handler.items
+ require.Len(t, handler.checkpoints, 1)
+ }
+
+ require.Equal(t, "from=previous&to=head", compareQuery)
+ require.Zero(t, treeRequests)
+ require.Len(t, items, 4)
+ require.True(t, items[0].IsDeleted)
+ require.Equal(t, "removed.md", items[0].Metadata["gitlab_path"])
+ require.True(t, items[1].IsDeleted)
+ require.Equal(t, "old.md", items[1].Metadata["gitlab_path"])
+ require.False(t, items[2].IsDeleted)
+ require.Equal(t, "renamed.md", items[2].Metadata["gitlab_path"])
+ require.False(t, items[3].IsDeleted)
+ require.Equal(t, "guide.md", items[3].Metadata["gitlab_path"])
+ require.Equal(t, "head", next.ConnectorCursor["projects"].(map[string]string)["1"])
+ })
+ }
+}