Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix qdrant collection matching (#3089)
Mohamed Arbic100c07fde34
diff --git a/internal/application/repository/retriever/qdrant/repository.go b/internal/application/repository/retriever/qdrant/repository.goindex ffbdd20fe..cc4ed2a79 100644--- a/internal/application/repository/retriever/qdrant/repository.go+++ b/internal/application/repository/retriever/qdrant/repository.go@@ -4,6 +4,7 @@ import ("context""errors""fmt"+ "strconv""strings""unicode/utf8"@@ -55,6 +56,16 @@ func (q *qdrantRepository) getCollectionName(dimension int) string {return fmt.Sprintf("%s_%d", q.collectionBaseName, dimension)}+// matchesCollectionName accepts only names generated for a positive dimension.+func (q *qdrantRepository) matchesCollectionName(collectionName string) bool {+ suffix, ok := strings.CutPrefix(collectionName, q.collectionBaseName+"_")+ if !ok {+ return false+ }+ dimension, err := strconv.Atoi(suffix)+ return err == nil && dimension > 0 && suffix == strconv.Itoa(dimension)+}+// collectionExists reports whether the dimension-specific collection is// present. Collections are created lazily on the first write for a dimension,// so callers that only need to know whether there is anything to act on must@@ -502,9 +513,7 @@ func (q *qdrantRepository) BatchUpdateChunkEnabledStatus(ctx context.Context, chvar updateErr error// Update in all matching collectionsfor _, collectionName := range collections {- // Only process collections that start with our base name- if len(collectionName) <= len(q.collectionBaseName) ||- collectionName[:len(q.collectionBaseName)] != q.collectionBaseName {+ if !q.matchesCollectionName(collectionName) {continue}@@ -591,9 +600,7 @@ func (q *qdrantRepository) BatchUpdateChunkTagID(ctx context.Context, chunkTagMavar updateErr error// Update in all matching collectionsfor _, collectionName := range collections {- // Only process collections that start with our base name- if len(collectionName) <= len(q.collectionBaseName) ||- collectionName[:len(q.collectionBaseName)] != q.collectionBaseName {+ if !q.matchesCollectionName(collectionName) {continue}@@ -791,9 +798,7 @@ func (q *qdrantRepository) KeywordsRetrieve(ctx context.Context,// Search in all matching collectionsfor _, collectionName := range collections {log.Debugf("[Qdrant] Checking collection: %s", collectionName)- // Only process collections that start with our base name- if len(collectionName) <= len(q.collectionBaseName) ||- collectionName[:len(q.collectionBaseName)] != q.collectionBaseName {+ if !q.matchesCollectionName(collectionName) {log.Debugf("[Qdrant] Skipping collection %s (doesn't match base name %s)", collectionName, q.collectionBaseName)continue}diff --git a/internal/application/repository/retriever/qdrant/repository_test.go b/internal/application/repository/retriever/qdrant/repository_test.goindex 1359ee88f..28a477494 100644--- a/internal/application/repository/retriever/qdrant/repository_test.go+++ b/internal/application/repository/retriever/qdrant/repository_test.go@@ -7,6 +7,7 @@ import ("fmt""os""slices"+ "strconv""strings""testing""time"@@ -20,6 +21,60 @@ import ("google.golang.org/grpc/status")+func TestMatchesCollectionName(t *testing.T) {+ tests := []struct {+ name string+ base string+ collection string+ want bool+ }{+ {"minimum dimension", "embeddings", "embeddings_1", true},+ {"typical dimension", "weknora_embeddings", "weknora_embeddings_768", true},+ {"custom base", "team_42_vectors", "team_42_vectors_1536", true},+ {"base ending in underscore", "vectors_", "vectors__768", true},+ {"longer base", "vectors", "vectors_extra_768", false},+ {"backup collection", "weknora_embeddings", "weknora_embeddings_backup_768", false},+ {"backup suffix", "vectors", "vectors_768_backup", false},+ {"missing separator", "vectors", "vectors768", false},+ {"missing dimension", "vectors", "vectors_", false},+ {"base only", "vectors", "vectors", false},+ {"short name", "vectors", "vec", false},+ {"empty name", "vectors", "", false},+ {"different base", "vectors", "other_768", false},+ {"case mismatch", "vectors", "Vectors_768", false},+ {"zero dimension", "vectors", "vectors_0", false},+ {"negative dimension", "vectors", "vectors_-768", false},+ {"explicit plus", "vectors", "vectors_+768", false},+ {"leading zero", "vectors", "vectors_0768", false},+ {"fraction", "vectors", "vectors_7.68", false},+ {"non decimal", "vectors", "vectors_0x300", false},+ {"leading space", "vectors", "vectors_ 768", false},+ {"trailing space", "vectors", "vectors_768 ", false},+ {"unicode digits", "vectors", "vectors_768", false},+ {"overflow", "vectors", "vectors_999999999999999999999999999999999", false},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ repo := &qdrantRepository{collectionBaseName: tt.base}+ if got := repo.matchesCollectionName(tt.collection); got != tt.want {+ t.Errorf("matchesCollectionName(%q) = %v, want %v", tt.collection, got, tt.want)+ }+ })+ }+}++func TestMatchesGeneratedCollectionNames(t *testing.T) {+ repo := &qdrantRepository{collectionBaseName: "custom_vectors"}+ for _, dimension := range []int{1, 384, 768, 1536, int(^uint(0) >> 1)} {+ t.Run(strconv.Itoa(dimension), func(t *testing.T) {+ name := repo.getCollectionName(dimension)+ if !repo.matchesCollectionName(name) {+ t.Errorf("generated collection name %q was rejected", name)+ }+ })+ }+}+func TestNewQdrantValueMapSanitizesInvalidUTF8AndNUL(t *testing.T) {malformed := "prefix" + string([]byte{0xff}) + "\x00suffix"payload := newQdrantValueMap(map[string]any{