Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(datasource): preserve distinct GitLab files during deduplication

wizardchen1f82fbc9b5e6
diff --git a/internal/application/repository/knowledge.go b/internal/application/repository/knowledge.go
index db94133e7..db0e1f7d7 100644
--- a/internal/application/repository/knowledge.go
+++ b/internal/application/repository/knowledge.go
@@ -367,6 +367,10 @@ func (r *knowledgeRepository) CheckKnowledgeExists(
switch params.Type {
case "file":
+ if params.DataSourceID != "" && params.ExternalID != "" {
+ query = query.Where("metadata->>'datasource_id' = ? AND metadata->>'external_id' = ?",
+ params.DataSourceID, params.ExternalID)
+ }
// File content is only a duplicate within the same file type. This keeps
// same-content documents with distinct formats (for example, .md and
// .txt) available as separate knowledge items.
diff --git a/internal/application/repository/knowledge_duplicate_test.go b/internal/application/repository/knowledge_duplicate_test.go
index f7279d046..57e43a37b 100644
--- a/internal/application/repository/knowledge_duplicate_test.go
+++ b/internal/application/repository/knowledge_duplicate_test.go
@@ -61,3 +61,41 @@ func TestCheckKnowledgeExists_FileHashIsScopedByFileType(t *testing.T) {
assert.Equal(t, "md", knowledge.FileType)
})
}
+
+func TestCheckKnowledgeExists_FileSourceIdentity(t *testing.T) {
+ db := setupKnowledgeTestDB(t)
+ repo := NewKnowledgeRepository(db)
+ const metadata = `{"datasource_id":"ds-1","external_id":"gitlab:1:main:README.md"}`
+ require.NoError(t, db.Exec(`
+ INSERT INTO knowledges (
+ id, tenant_id, knowledge_base_id, type, file_name, file_type, file_size, file_hash, parse_status, metadata
+ )
+ VALUES (?, 1, 'kb-1', 'file', 'README.md', 'md', 10, 'same-hash', 'completed', ?)
+ `, uuid.NewString(), metadata).Error)
+ for _, tc := range []struct {
+ name, source, externalID, hash, fileType string
+ want bool
+ }{
+ {"same source item", "ds-1", "gitlab:1:main:README.md", "same-hash", "md", true},
+ {"nested path", "ds-1", "gitlab:1:main:a/b/c/README.md", "same-hash", "md", false},
+ {"another branch", "ds-1", "gitlab:1:release:README.md", "same-hash", "md", false},
+ {"another source", "ds-2", "gitlab:1:main:README.md", "same-hash", "md", false},
+ {"updated content", "ds-1", "gitlab:1:main:README.md", "new-hash", "md", false},
+ {"another format", "ds-1", "gitlab:1:main:README.md", "same-hash", "txt", false},
+ {"case insensitive format", "ds-1", "gitlab:1:main:README.md", "same-hash", "MD", true},
+ {"unscoped upload", "", "", "same-hash", "md", true},
+ {"missing source", "", "new-path", "same-hash", "md", true},
+ {"missing external ID", "ds-2", "", "same-hash", "md", true},
+ {"filename fallback same item", "ds-1", "gitlab:1:main:README.md", "", "md", true},
+ {"filename fallback nested item", "ds-1", "gitlab:1:main:a/b/c/README.md", "", "md", false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ exists, _, err := repo.CheckKnowledgeExists(context.Background(), 1, "kb-1", &types.KnowledgeCheckParams{
+ Type: "file", FileName: "README.md", FileSize: 10, FileHash: tc.hash, FileType: tc.fileType,
+ DataSourceID: tc.source, ExternalID: tc.externalID,
+ })
+ require.NoError(t, err)
+ require.Equal(t, tc.want, exists)
+ })
+ }
+}
diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go
index d294f2c14..40373f563 100644
--- a/internal/application/service/knowledge_create.go
+++ b/internal/application/service/knowledge_create.go
@@ -87,13 +87,21 @@ func (s *knowledgeService) CreateKnowledgeFromFile(ctx context.Context,
// Check if file already exists
tenantID := ctx.Value(types.TenantIDContextKey).(uint64)
logger.Infof(ctx, "Checking if file exists, tenant ID: %d", tenantID)
- exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, &types.KnowledgeCheckParams{
+ checkParams := &types.KnowledgeCheckParams{
Type: "file",
FileName: fileName,
FileType: getFileType(fileName),
FileSize: file.Size,
FileHash: hash,
- })
+ }
+ // Repository paths are independent source files, even when their bytes are
+ // identical (for example, README templates in different subdirectories).
+ // Keep retries deduplicated within the same GitLab data source and path.
+ if channel == types.ConnectorTypeGitLab {
+ checkParams.DataSourceID = metadata["datasource_id"]
+ checkParams.ExternalID = metadata["external_id"]
+ }
+ exists, existingKnowledge, err := s.repo.CheckKnowledgeExists(ctx, tenantID, kbID, checkParams)
if err != nil {
logger.Errorf(ctx, "Failed to check knowledge existence: %v", err)
return nil, err
diff --git a/internal/application/service/knowledge_create_gitlab_test.go b/internal/application/service/knowledge_create_gitlab_test.go
new file mode 100644
index 000000000..b805c775f
--- /dev/null
+++ b/internal/application/service/knowledge_create_gitlab_test.go
@@ -0,0 +1,79 @@
+package service
+
+import (
+ "context"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/application/repository"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
+)
+
+type gitLabKnowledgeRepo struct {
+ interfaces.KnowledgeRepository
+}
+
+func (r *gitLabKnowledgeRepo) GetKnowledgeTags(context.Context, []string) (map[string][]*types.KnowledgeTag, error) {
+ return map[string][]*types.KnowledgeTag{}, nil
+}
+
+func TestCreateKnowledgeFromFileGitLabPreservesSourcePaths(t *testing.T) {
+ db := setupKnowledgeSharedAccessDB(t)
+ sqlDB, err := db.DB()
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = sqlDB.Close() })
+ repo := &gitLabKnowledgeRepo{KnowledgeRepository: repository.NewKnowledgeRepository(db)}
+ svc := &knowledgeService{
+ repo: repo,
+ kbService: &createKnowledgeFileKBServiceStub{kb: &types.KnowledgeBase{ID: "kb-1"}},
+ fileSvc: &createKnowledgeFileServiceStub{},
+ task: &createKnowledgeTaskEnqueuerStub{},
+ }
+ ctx := newCreateKnowledgeFileContext()
+ create := func(channel, source, file string) (*types.Knowledge, error) {
+ t.Helper()
+ metadata := map[string]string{}
+ if source != "" {
+ metadata["datasource_id"] = source
+ metadata["external_id"] = "gitlab:instance:1:main:" + file
+ }
+ return svc.CreateKnowledgeFromFile(ctx, "kb-1", newMultipartFileHeader(t, "README.md", "# same content"),
+ metadata, nil, "docs-main/"+file, nil, channel, nil)
+ }
+
+ root, err := create(types.ConnectorTypeGitLab, "ds-1", "README.md")
+ require.NoError(t, err)
+ ds := &types.DataSource{ID: "ds-1", TenantID: 1, KnowledgeBaseID: "kb-1", Type: types.ConnectorTypeGitLab}
+ item := &types.FetchedItem{
+ ExternalID: "gitlab:instance:1:main:a/b/c/d/e/README.md",
+ FileName: "docs-main/a/b/c/d/e/README.md",
+ Content: []byte("# same content"),
+ }
+ result := &types.SyncResult{}
+ (&DataSourceService{knowledgeService: svc}).applyFetchedItem(ctx, ds, item, nil, result)
+ require.Equal(t, 1, result.Created, "identical content at a different repository path must still be imported")
+ require.Zero(t, result.Skipped)
+ require.Zero(t, result.Failed)
+ nested, err := repo.FindByDataSourceExternalID(ctx, ds.TenantID, ds.KnowledgeBaseID, ds.ID, item.ExternalID)
+ require.NoError(t, err)
+ require.NotNil(t, nested)
+ require.NotEqual(t, root.ID, nested.ID)
+ require.Equal(t, "docs-main/a/b/c/d/e", nested.FolderPath)
+
+ otherSource, err := create(types.ConnectorTypeGitLab, "ds-2", "README.md")
+ require.NoError(t, err, "another data source must retain its own file")
+ require.NotEqual(t, root.ID, otherSource.ID)
+
+ duplicate, err := create(types.ConnectorTypeGitLab, "ds-1", "a/b/c/d/e/README.md")
+ var dupErr *types.DuplicateKnowledgeError
+ require.ErrorAs(t, err, &dupErr)
+ require.Equal(t, nested.ID, duplicate.ID, "a retry must match this path's own knowledge")
+
+ for _, channel := range []string{"", types.ConnectorTypeFeishu} {
+ _, err := create(channel, "ds-3", "another/README.md")
+ require.ErrorAs(t, err, &dupErr, "other import channels retain content deduplication")
+ }
+ _, err = create(types.ConnectorTypeGitLab, "", "missing-identity/README.md")
+ require.ErrorAs(t, err, &dupErr, "incomplete source identity retains content deduplication")
+}
diff --git a/internal/datasource/connector/gitlab/connector_recursion_test.go b/internal/datasource/connector/gitlab/connector_recursion_test.go
new file mode 100644
index 000000000..f3d82bac3
--- /dev/null
+++ b/internal/datasource/connector/gitlab/connector_recursion_test.go
@@ -0,0 +1,109 @@
+package gitlab
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/http/httptest"
+ "path"
+ "slices"
+ "strings"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/stretchr/testify/require"
+)
+
+// Exercise the real tree and raw-file APIs, including a subdirectory returned
+// on the second page. Browsing remains shallow; fetching must visit every level.
+func TestConnectorFetchesNestedDirectories(t *testing.T) {
+ for _, mode := range []string{
+ "all", "incremental", "stream", "incremental compare fallback", "stream compare fallback",
+ } {
+ t.Run(mode, func(t *testing.T) {
+ allowLocalGitLabServer(t)
+ dirs := []string{"docs", "docs/a", "docs/a/b", "docs/a/b/c", "docs/a/b/c/d"}
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ switch r.URL.Path {
+ case "/api/v4/projects/1":
+ _, _ = w.Write([]byte(`{
+ "id":1,"name":"docs","path_with_namespace":"group/docs","default_branch":"main"
+ }`))
+ case "/api/v4/projects/1/repository/commits/main":
+ _, _ = w.Write([]byte(`{"id":"head"}`))
+ case "/api/v4/projects/1/repository/compare":
+ _, _ = w.Write([]byte(`{"compare_timeout":true}`))
+ case "/api/v4/projects/1/repository/tree":
+ dir := r.URL.Query().Get("path")
+ index := slices.Index(dirs, dir)
+ if index < 0 || r.URL.Query().Get("ref") != "main" {
+ http.Error(w, "unexpected tree scope", http.StatusBadRequest)
+ return
+ }
+ var entries []treeEntry
+ switch r.URL.Query().Get("page") {
+ case "1":
+ entries = append(entries, treeEntry{Name: "README.md", Path: dir + "/README.md", Type: "blob"})
+ if index < len(dirs)-1 {
+ w.Header().Set("X-Next-Page", "2")
+ }
+ case "2":
+ if index < len(dirs)-1 {
+ child := dirs[index+1]
+ entries = append(entries, treeEntry{Name: path.Base(child), Path: child, Type: "tree"})
+ }
+ default:
+ http.Error(w, "unexpected page", http.StatusBadRequest)
+ return
+ }
+ _ = json.NewEncoder(w).Encode(entries)
+ default:
+ if strings.HasPrefix(r.URL.Path, "/api/v4/projects/1/repository/files/docs/") &&
+ strings.HasSuffix(r.URL.Path, "/raw") {
+ _, _ = fmt.Fprint(w, "# same content")
+ return
+ }
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+ cfg := &types.DataSourceConfig{
+ Credentials: map[string]interface{}{"base_url": server.URL, "access_token": "token"},
+ Settings: map[string]interface{}{"projects": []interface{}{
+ map[string]interface{}{"project_id": "1", "paths": []interface{}{"docs", "docs/a"}},
+ }},
+ }
+ connector := NewConnector()
+ var old *types.SyncCursor
+ if strings.Contains(mode, "fallback") {
+ old = gitLabCursor(cursor{Projects: map[string]string{"1": "previous"}})
+ }
+ var items []types.FetchedItem
+ var err error
+ switch {
+ case mode == "all":
+ items, err = connector.FetchAll(context.Background(), cfg, nil)
+ case strings.HasPrefix(mode, "incremental"):
+ items, _, err = connector.FetchIncremental(context.Background(), cfg, old)
+ default:
+ handler := &gitLabStreamRecorder{}
+ _, err = connector.FetchStream(context.Background(), cfg, old, handler)
+ items = handler.items
+ require.Len(t, handler.checkpoints, 1)
+ }
+ require.NoError(t, err)
+ require.Len(t, items, len(dirs), "overlapping selected roots must not duplicate files")
+ for i, dir := range dirs {
+ require.Equal(t, "docs-main/"+dir+"/README.md", items[i].FileName)
+ require.Equal(t, dir+"/README.md", items[i].Metadata["gitlab_path"])
+ }
+
+ resources, err := connector.ListResources(context.Background(), cfg, "1:docs")
+ require.NoError(t, err)
+ require.Len(t, resources, 1)
+ require.Equal(t, "1:docs/a", resources[0].ExternalID)
+ })
+ }
+}
diff --git a/internal/types/knowledge.go b/internal/types/knowledge.go
index 6253423ad..dceb8e424 100644
--- a/internal/types/knowledge.go
+++ b/internal/types/knowledge.go
@@ -488,6 +488,9 @@ type KnowledgeCheckParams struct {
FileType string
FileSize int64
FileHash string
+ // When both are set, file deduplication is scoped to this source item.
+ DataSourceID string
+ ExternalID string
// URL parameters
URL string
// Text passage parameters