Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

9.2 KB
Raw Blame
package gitlab

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"path"
	"slices"
	"strings"
	"testing"

	"github.com/Tencent/WeKnora/internal/types"
	"github.com/stretchr/testify/require"
)

// Exercise the real tree and raw-file APIs, including a subdirectory returned
// on the second page. Browsing remains shallow; fetching must visit every level.
func TestConnectorFetchesNestedDirectories(t *testing.T) {
	for _, mode := range []string{
		"all", "incremental", "stream",
	} {
		t.Run(mode, func(t *testing.T) {
			allowLocalGitLabServer(t)
			dirs := []string{"docs", "docs/a", "docs/a/b", "docs/a/b/c", "docs/a/b/c/d"}
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Content-Type", "application/json")
				switch r.URL.Path {
				case "/api/v4/projects/1":
					_, _ = w.Write([]byte(`{
						"id":1,"name":"docs","path_with_namespace":"group/docs","default_branch":"main"
					}`))
				case "/api/v4/projects/1/repository/commits/main":
					_, _ = w.Write([]byte(`{"id":"head"}`))
				case "/api/v4/projects/1/repository/compare":
					_, _ = w.Write([]byte(`{"compare_timeout":true}`))
				case "/api/v4/projects/1/repository/tree":
					dir := r.URL.Query().Get("path")
					index := slices.Index(dirs, dir)
					if index < 0 || r.URL.Query().Get("ref") != "main" {
						http.Error(w, "unexpected tree scope", http.StatusBadRequest)
						return
					}
					var entries []treeEntry
					switch r.URL.Query().Get("page") {
					case "1":
						entries = append(entries, treeEntry{Name: "README.md", Path: dir + "/README.md", Type: "blob"})
						if index < len(dirs)-1 {
							w.Header().Set("X-Next-Page", "2")
						}
					case "2":
						if index < len(dirs)-1 {
							child := dirs[index+1]
							entries = append(entries, treeEntry{Name: path.Base(child), Path: child, Type: "tree"})
						}
					default:
						http.Error(w, "unexpected page", http.StatusBadRequest)
						return
					}
					_ = json.NewEncoder(w).Encode(entries)
				default:
					if strings.HasPrefix(r.URL.Path, "/api/v4/projects/1/repository/files/docs/") &&
						strings.HasSuffix(r.URL.Path, "/raw") {
						_, _ = fmt.Fprint(w, "# same content")
						return
					}
					http.NotFound(w, r)
				}
			}))
			defer server.Close()
			cfg := &types.DataSourceConfig{
				Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"},
				Settings: map[string]interface{}{"projects": []interface{}{
					map[string]interface{}{"project_id": "1", "paths": []interface{}{"docs", "docs/a"}},
				}},
			}
			connector := NewConnector()
			var old *types.SyncCursor
			if strings.Contains(mode, "fallback") {
				old = gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}})
			}
			var items []types.FetchedItem
			var err error
			switch {
			case mode == "all":
				items, err = connector.FetchAll(context.Background(), cfg, nil)
			case strings.HasPrefix(mode, "incremental"):
				items, _, err = connector.FetchIncremental(context.Background(), cfg, old)
			default:
				handler := &gitLabStreamRecorder{}
				_, err = connector.FetchStream(context.Background(), cfg, old, handler)
				items = handler.items
				require.Len(t, handler.checkpoints, 1)
			}
			require.NoError(t, err)
			require.Len(t, items, len(dirs), "overlapping selected roots must not duplicate files")
			for i, dir := range dirs {
				require.Equal(t, "docs-main/"+dir+"/README.md", items[i].FileName)
				require.Equal(t, dir+"/README.md", items[i].Metadata["gitlab_path"])
			}

			resources, err := connector.ListResources(context.Background(), cfg, "1:docs")
			require.NoError(t, err)
			require.Len(t, resources, 1)
			require.Equal(t, "1:docs/a", resources[0].ExternalID)
		})
	}
}

func TestConnectorCompareFailuresDoNotFallBackToFullSync(t *testing.T) {
	for _, failure := range []struct {
		name       string
		statusCode int
		body       string
	}{
		{name: "timeout response", statusCode: http.StatusOK, body: `{"compare_timeout":true}`},
		{name: "API error", statusCode: http.StatusInternalServerError, body: `{"message":"compare failed"}`},
	} {
		for _, mode := range []string{"incremental", "stream"} {
			t.Run(failure.name+" "+mode, func(t *testing.T) {
				allowLocalGitLabServer(t)
				downstreamRequests := 0
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					w.Header().Set("Content-Type", "application/json")
					switch r.URL.Path {
					case "/api/v4/projects/1":
						_, _ = w.Write([]byte(`{
							"id":1,"name":"docs","path_with_namespace":"group/docs","default_branch":"main"
						}`))
					case "/api/v4/projects/1/repository/commits/main":
						_, _ = w.Write([]byte(`{"id":"head"}`))
					case "/api/v4/projects/1/repository/compare":
						w.WriteHeader(failure.statusCode)
						_, _ = w.Write([]byte(failure.body))
					case "/api/v4/projects/1/repository/tree":
						downstreamRequests++
						_, _ = w.Write([]byte(`[{"name":"README.md","path":"README.md","type":"blob"}]`))
					default:
						if strings.Contains(r.URL.EscapedPath(), "/repository/files/") {
							downstreamRequests++
							_, _ = w.Write([]byte("# unchanged"))
							return
						}
						http.NotFound(w, r)
					}
				}))
				defer server.Close()

				cfg := &types.DataSourceConfig{
					Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"},
					Settings: map[string]interface{}{"projects": []interface{}{
						map[string]interface{}{"project_id": "1", "paths": []interface{}{}},
					}},
				}
				old := gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}})
				connector := NewConnector()

				switch mode {
				case "incremental":
					items, next, err := connector.FetchIncremental(context.Background(), cfg, old)
					require.ErrorContains(t, err, "gitlab compare project 1")
					require.Empty(t, items)
					require.Nil(t, next)
				case "stream":
					handler := &gitLabStreamRecorder{}
					next, err := connector.FetchStream(context.Background(), cfg, old, handler)
					require.ErrorContains(t, err, "gitlab compare project 1")
					require.Nil(t, next)
					require.Empty(t, handler.items)
					require.Empty(t, handler.checkpoints)
				}
				require.Zero(t, downstreamRequests, "compare failures must not trigger a full repository walk")
			})
		}
	}
}

func TestConnectorSuccessfulCompareStillEmitsOnlyChangedFiles(t *testing.T) {
	for _, mode := range []string{"incremental", "stream"} {
		t.Run(mode, func(t *testing.T) {
			allowLocalGitLabServer(t)
			var compareQuery string
			treeRequests := 0
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("Content-Type", "application/json")
				switch r.URL.Path {
				case "/api/v4/projects/1":
					_, _ = w.Write([]byte(`{
						"id":1,"name":"docs","path_with_namespace":"group/docs",
						"web_url":"https://gitlab.test/group/docs","default_branch":"main"
					}`))
				case "/api/v4/projects/1/repository/commits/main":
					_, _ = w.Write([]byte(`{"id":"head"}`))
				case "/api/v4/projects/1/repository/compare":
					compareQuery = r.URL.RawQuery
					_, _ = w.Write([]byte(`{"diffs":[
						{"old_path":"removed.md","new_path":"removed.md","deleted_file":true},
						{"old_path":"old.md","new_path":"renamed.md","renamed_file":true},
						{"old_path":"guide.md","new_path":"guide.md"},
						{"old_path":"server.go","new_path":"server.go"}
					]}`))
				case "/api/v4/projects/1/repository/tree":
					treeRequests++
					http.Error(w, "unexpected full repository walk", http.StatusInternalServerError)
				default:
					if strings.Contains(r.URL.EscapedPath(), "/repository/files/") {
						_, _ = w.Write([]byte("# changed"))
						return
					}
					http.NotFound(w, r)
				}
			}))
			defer server.Close()

			cfg := &types.DataSourceConfig{
				Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"},
				Settings: map[string]interface{}{"projects": []interface{}{
					map[string]interface{}{"project_id": "1", "paths": []interface{}{}},
				}},
			}
			old := gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}})
			connector := NewConnector()
			var items []types.FetchedItem
			var next *types.SyncCursor

			switch mode {
			case "incremental":
				var err error
				items, next, err = connector.FetchIncremental(context.Background(), cfg, old)
				require.NoError(t, err)
			case "stream":
				handler := &gitLabStreamRecorder{}
				var err error
				next, err = connector.FetchStream(context.Background(), cfg, old, handler)
				require.NoError(t, err)
				items = handler.items
				require.Len(t, handler.checkpoints, 1)
			}

			require.Equal(t, "from=previous&straight=true&to=head", compareQuery)
			require.Zero(t, treeRequests)
			require.Len(t, items, 4)
			require.True(t, items[0].IsDeleted)
			require.Equal(t, "removed.md", items[0].Metadata["gitlab_path"])
			require.True(t, items[1].IsDeleted)
			require.Equal(t, "old.md", items[1].Metadata["gitlab_path"])
			require.False(t, items[2].IsDeleted)
			require.Equal(t, "renamed.md", items[2].Metadata["gitlab_path"])
			require.False(t, items[3].IsDeleted)
			require.Equal(t, "guide.md", items[3].Metadata["gitlab_path"])
			require.Equal(t, "head", next.ConnectorCursor["projects"].(map[string]string)["1"])
		})
	}
}