Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(datasource): preserve distinct GitLab files during deduplication
wizardchen1f82fbc9b5e6
diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.goindex db94133e7..db0e1f7d7 100644--- a/internal/application/repository/knowledge.go+++ b/internal/application/repository/knowledge.go@@ -367,6 +367,10 @@ func (r *knowledgeRepository) CheckKnowledgeExists(switch params.Type {case "file":+ if params.DataSourceID != "" && params.ExternalID != "" {+ query = query.Where("metadata->>'datasource_id' = ? AND metadata->>'external_id' = ?",+ params.DataSourceID, params.ExternalID)+ }// File content is only a duplicate within the same file type. This keeps// same-content documents with distinct formats (for example, .md and// .txt) available as separate knowledge items.diff --git a/internal/application/repository/knowledge_duplicate_test.go b/internal/application/repository/knowledge_duplicate_test.goindex f7279d046..57e43a37b 100644--- a/internal/application/repository/knowledge_duplicate_test.go+++ b/internal/application/repository/knowledge_duplicate_test.go@@ -61,3 +61,41 @@ func TestCheckKnowledgeExists_FileHashIsScopedByFileType(t *testing.T) {assert.Equal(t, "md", knowledge.FileType)})}++func TestCheckKnowledgeExists_FileSourceIdentity(t *testing.T) {+ db := setupKnowledgeTestDB(t)+ repo := NewKnowledgeRepository(db)+ const metadata = `{"datasource_id":"ds-1","external_id":"gitlab:1:main:README.md"}`+ require.NoError(t, db.Exec(`+ INSERT INTO knowledges (+ id, tenant_id, knowledge_base_id, type, file_name, file_type, file_size, file_hash, parse_status, metadata+ )+ VALUES (?, 1, 'kb-1', 'file', 'README.md', 'md', 10, 'same-hash', 'completed', ?)+ `, uuid.NewString(), metadata).Error)+ for _, tc := range []struct {+ name, source, externalID, hash, fileType string+ want bool+ }{+ {"same source item", "ds-1", "gitlab:1:main:README.md", "same-hash", "md", true},+ {"nested path", "ds-1", "gitlab:1:main:a/b/c/README.md", "same-hash", "md", false},+ {"another branch", "ds-1", "gitlab:1:release:README.md", "same-hash", "md", false},+ {"another source", "ds-2", "gitlab:1:main:README.md", "same-hash", "md", false},+ {"updated content", "ds-1", "gitlab:1:main:README.md", "new-hash", "md", false},+ {"another format", "ds-1", "gitlab:1:main:README.md", "same-hash", "txt", false},+ {"case insensitive format", "ds-1", "gitlab:1:main:README.md", "same-hash", "MD", true},+ {"unscoped upload", "", "", "same-hash", "md", true},+ {"missing source", "", "new-path", "same-hash", "md", true},+ {"missing external ID", "ds-2", "", "same-hash", "md", true},+ {"filename fallback same item", "ds-1", "gitlab:1:main:README.md", "", "md", true},+ {"filename fallback nested item", "ds-1", "gitlab:1:main:a/b/c/README.md", "", "md", false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ exists, _, err := repo.CheckKnowledgeExists(context.Background(), 1, "kb-1", &types.KnowledgeCheckParams{+ Type: "file", FileName: "README.md", FileSize: 10, FileHash: tc.hash, FileType: tc.fileType,+ DataSourceID: tc.source, ExternalID: tc.externalID,+ })+ require.NoError(t, err)+ require.Equal(t, tc.want, exists)+ })+ }+}diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.goindex d294f2c14..40373f563 100644--- a/internal/application/service/knowledge_create.go+++ b/internal/application/service/knowledge_create.go@@ -87,13 +87,21 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,// Check if file already existstenantID := ctx.Value(types.TenantIDContextKey).(uint64)logger.Infof(ctx, "Checking if file exists, tenant ID: %d", tenantID)- exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{+ checkParams := &types.KnowledgeCheckParams{Type: "file",FileName: fileName,FileType: getFileType(fileName),FileSize: file.Size,FileHash: hash,- })+ }+ // Repository paths are independent source files, even when their bytes are+ // identical (for example, README templates in different subdirectories).+ // Keep retries deduplicated within the same GitLab data source and path.+ if channel == types.ConnectorTypeGitLab {+ checkParams.DataSourceID = metadata["datasource_id"]+ checkParams.ExternalID = metadata["external_id"]+ }+ exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, checkParams)if err != nil {logger.Errorf(ctx, "Failed to check knowledge existence: %v", err)return nil, errdiff --git a/internal/application/service/knowledge_create_gitlab_test.go b/internal/application/service/knowledge_create_gitlab_test.gonew file mode 100644index 000000000..b805c775f--- /dev/null+++ b/internal/application/service/knowledge_create_gitlab_test.go@@ -0,0 +1,79 @@+package service++import (+ "context"+ "testing"++ "github.com/Tencent/WeKnora/internal/application/repository"+ "github.com/Tencent/WeKnora/internal/types"+ "github.com/Tencent/WeKnora/internal/types/interfaces"+ "github.com/stretchr/testify/require"+)++type gitLabKnowledgeRepo struct {+ interfaces.KnowledgeRepository+}++func (r *gitLabKnowledgeRepo) GetKnowledgeTags(context.Context, []string) (map[string][]*types.KnowledgeTag, error) {+ return map[string][]*types.KnowledgeTag{}, nil+}++func TestCreateKnowledgeFromFileGitLabPreservesSourcePaths(t *testing.T) {+ db := setupKnowledgeSharedAccessDB(t)+ sqlDB, err := db.DB()+ require.NoError(t, err)+ t.Cleanup(func() { _ = sqlDB.Close() })+ repo := &gitLabKnowledgeRepo{KnowledgeRepository: repository.NewKnowledgeRepository(db)}+ svc := &knowledgeService{+ repo: repo,+ kbService: &createKnowledgeFileKBServiceStub{kb: &types.KnowledgeBase{ID: "kb-1"}},+ fileSvc: &createKnowledgeFileServiceStub{},+ task: &createKnowledgeTaskEnqueuerStub{},+ }+ ctx := newCreateKnowledgeFileContext()+ create := func(channel, source, file string) (*types.Knowledge, error) {+ t.Helper()+ metadata := map[string]string{}+ if source != "" {+ metadata["datasource_id"] = source+ metadata["external_id"] = "gitlab:instance:1:main:" + file+ }+ return svc.CreateKnowledgeFromFile(ctx, "kb-1", newMultipartFileHeader(t, "README.md", "# same content"),+ metadata, nil, "docs-main/"+file, nil, channel, nil)+ }++ root, err := create(types.ConnectorTypeGitLab, "ds-1", "README.md")+ require.NoError(t, err)+ ds := &types.DataSource{ID: "ds-1", TenantID: 1, KnowledgeBaseID: "kb-1", Type: types.ConnectorTypeGitLab}+ item := &types.FetchedItem{+ ExternalID: "gitlab:instance:1:main:a/b/c/d/e/README.md",+ FileName: "docs-main/a/b/c/d/e/README.md",+ Content: []byte("# same content"),+ }+ result := &types.SyncResult{}+ (&DataSourceService{knowledgeService: svc}).applyFetchedItem(ctx, ds, item, nil, result)+ require.Equal(t, 1, result.Created, "identical content at a different repository path must still be imported")+ require.Zero(t, result.Skipped)+ require.Zero(t, result.Failed)+ nested, err := repo.FindByDataSourceExternalID(ctx, ds.TenantID, ds.KnowledgeBaseID, ds.ID, item.ExternalID)+ require.NoError(t, err)+ require.NotNil(t, nested)+ require.NotEqual(t, root.ID, nested.ID)+ require.Equal(t, "docs-main/a/b/c/d/e", nested.FolderPath)++ otherSource, err := create(types.ConnectorTypeGitLab, "ds-2", "README.md")+ require.NoError(t, err, "another data source must retain its own file")+ require.NotEqual(t, root.ID, otherSource.ID)++ duplicate, err := create(types.ConnectorTypeGitLab, "ds-1", "a/b/c/d/e/README.md")+ var dupErr *types.DuplicateKnowledgeError+ require.ErrorAs(t, err, &dupErr)+ require.Equal(t, nested.ID, duplicate.ID, "a retry must match this path's own knowledge")++ for _, channel := range []string{"", types.ConnectorTypeFeishu} {+ _, err := create(channel, "ds-3", "another/README.md")+ require.ErrorAs(t, err, &dupErr, "other import channels retain content deduplication")+ }+ _, err = create(types.ConnectorTypeGitLab, "", "missing-identity/README.md")+ require.ErrorAs(t, err, &dupErr, "incomplete source identity retains content deduplication")+}diff --git a/internal/datasource/connector/gitlab/connector_recursion_test.go b/internal/datasource/connector/gitlab/connector_recursion_test.gonew file mode 100644index 000000000..f3d82bac3--- /dev/null+++ b/internal/datasource/connector/gitlab/connector_recursion_test.go@@ -0,0 +1,109 @@+package gitlab++import (+ "context"+ "encoding/json"+ "fmt"+ "net/http"+ "net/http/httptest"+ "path"+ "slices"+ "strings"+ "testing"++ "github.com/Tencent/WeKnora/internal/types"+ "github.com/stretchr/testify/require"+)++// Exercise the real tree and raw-file APIs, including a subdirectory returned+// on the second page. Browsing remains shallow; fetching must visit every level.+func TestConnectorFetchesNestedDirectories(t *testing.T) {+ for _, mode := range []string{+ "all", "incremental", "stream", "incremental compare fallback", "stream compare fallback",+ } {+ t.Run(mode, func(t *testing.T) {+ allowLocalGitLabServer(t)+ dirs := []string{"docs", "docs/a", "docs/a/b", "docs/a/b/c", "docs/a/b/c/d"}+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ switch r.URL.Path {+ case "/api/v4/projects/1":+ _, _ = w.Write([]byte(`{+ "id":1,"name":"docs","path_with_namespace":"group/docs","default_branch":"main"+ }`))+ case "/api/v4/projects/1/repository/commits/main":+ _, _ = w.Write([]byte(`{"id":"head"}`))+ case "/api/v4/projects/1/repository/compare":+ _, _ = w.Write([]byte(`{"compare_timeout":true}`))+ case "/api/v4/projects/1/repository/tree":+ dir := r.URL.Query().Get("path")+ index := slices.Index(dirs, dir)+ if index < 0 || r.URL.Query().Get("ref") != "main" {+ http.Error(w, "unexpected tree scope", http.StatusBadRequest)+ return+ }+ var entries []treeEntry+ switch r.URL.Query().Get("page") {+ case "1":+ entries = append(entries, treeEntry{Name: "README.md", Path: dir + "/README.md", Type: "blob"})+ if index < len(dirs)-1 {+ w.Header().Set("X-Next-Page", "2")+ }+ case "2":+ if index < len(dirs)-1 {+ child := dirs[index+1]+ entries = append(entries, treeEntry{Name: path.Base(child), Path: child, Type: "tree"})+ }+ default:+ http.Error(w, "unexpected page", http.StatusBadRequest)+ return+ }+ _ = json.NewEncoder(w).Encode(entries)+ default:+ if strings.HasPrefix(r.URL.Path, "/api/v4/projects/1/repository/files/docs/") &&+ strings.HasSuffix(r.URL.Path, "/raw") {+ _, _ = fmt.Fprint(w, "# same content")+ return+ }+ http.NotFound(w, r)+ }+ }))+ defer server.Close()+ cfg := &types.DataSourceConfig{+ Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"},+ Settings: map[string]interface{}{"projects": []interface{}{+ map[string]interface{}{"project_id": "1", "paths": []interface{}{"docs", "docs/a"}},+ }},+ }+ connector := NewConnector()+ var old *types.SyncCursor+ if strings.Contains(mode, "fallback") {+ old = gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}})+ }+ var items []types.FetchedItem+ var err error+ switch {+ case mode == "all":+ items, err = connector.FetchAll(context.Background(), cfg, nil)+ case strings.HasPrefix(mode, "incremental"):+ items, _, err = connector.FetchIncremental(context.Background(), cfg, old)+ default:+ handler := &gitLabStreamRecorder{}+ _, err = connector.FetchStream(context.Background(), cfg, old, handler)+ items = handler.items+ require.Len(t, handler.checkpoints, 1)+ }+ require.NoError(t, err)+ require.Len(t, items, len(dirs), "overlapping selected roots must not duplicate files")+ for i, dir := range dirs {+ require.Equal(t, "docs-main/"+dir+"/README.md", items[i].FileName)+ require.Equal(t, dir+"/README.md", items[i].Metadata["gitlab_path"])+ }++ resources, err := connector.ListResources(context.Background(), cfg, "1:docs")+ require.NoError(t, err)+ require.Len(t, resources, 1)+ require.Equal(t, "1:docs/a", resources[0].ExternalID)+ })+ }+}diff --git a/internal/types/knowledge.go b/internal/types/knowledge.goindex 6253423ad..dceb8e424 100644--- a/internal/types/knowledge.go+++ b/internal/types/knowledge.go@@ -488,6 +488,9 @@ type KnowledgeCheckParams struct {FileType stringFileSize int64FileHash string+ // When both are set, file deduplication is scoped to this source item.+ DataSourceID string+ ExternalID string// URL parametersURL string// Text passage parameters