Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(retriever/es): keep vectors and enabled state on CopyIndices (#3678)
lyingbug65d8129cb64f
diff --git a/internal/application/repository/retriever/elasticsearch/v7/copy_test.go b/internal/application/repository/retriever/elasticsearch/v7/copy_test.gonew file mode 100644index 000000000..1cc4aa1af--- /dev/null+++ b/internal/application/repository/retriever/elasticsearch/v7/copy_test.go@@ -0,0 +1,145 @@+package v7++import (+ "bufio"+ "bytes"+ "context"+ "encoding/json"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "testing"++ "github.com/elastic/go-elasticsearch/v7"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// copySourceHits covers every source_id shape CopyIndices remaps: a regular+// chunk row, a generated-question row sharing that chunk, a row with an+// unrelated source_id (remapped to a fresh uuid), and a legacy row written+// before is_enabled existed.+const copySourceHits = `[+ {"_index":"vectors","_id":"d1","_score":1,"_source":{"content":"chunk","source_id":"c1","source_type":0,+ "chunk_id":"c1","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.1,0.2],"is_enabled":true}},+ {"_index":"vectors","_id":"d2","_score":1,"_source":{"content":"question","source_id":"c1-q1","source_type":1,+ "chunk_id":"c1","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.3,0.4],"is_enabled":true}},+ {"_index":"vectors","_id":"d3","_score":1,"_source":{"content":"summary","source_id":"img-9","source_type":2,+ "chunk_id":"c2","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.5,0.6],"is_enabled":false}},+ {"_index":"vectors","_id":"d4","_score":1,"_source":{"content":"legacy","source_id":"c3","source_type":0,+ "chunk_id":"c3","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.7,0.8]}}+]`++type copiedDoc struct {+ Content string `json:"content"`+ SourceID string `json:"source_id"`+ SourceType int `json:"source_type"`+ ChunkID string `json:"chunk_id"`+ KnowledgeID string `json:"knowledge_id"`+ KnowledgeBaseID string `json:"knowledge_base_id"`+ Embedding []float32 `json:"embedding"`+ IsEnabled *bool `json:"is_enabled"`+}++// fakeCopyServer serves one page of copySourceHits and records the search+// query and every document sent through _bulk.+type fakeCopyServer struct {+ mu sync.Mutex+ searchQuery string+ docs []copiedDoc+}++func (f *fakeCopyServer) handler(t *testing.T) http.HandlerFunc {+ return func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ body, _ := io.ReadAll(r.Body)+ f.mu.Lock()+ defer f.mu.Unlock()+ switch r.URL.Path {+ case "/":+ _, _ = w.Write([]byte(`{"version":{"number":"7.17.0","build_flavor":"default"},` ++ `"tagline":"You Know, for Search"}`))+ case "/vectors/_search":+ f.searchQuery = string(body)+ _, _ = w.Write([]byte(`{"took":1,"timed_out":false,` ++ `"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},` ++ `"hits":{"total":{"value":4,"relation":"eq"},"hits":` + copySourceHits + `}}`))+ case "/vectors/_bulk":+ sc := bufio.NewScanner(bytes.NewReader(body))+ sc.Buffer(make([]byte, 0, 1<<20), 1<<20)+ for sc.Scan() {+ line := sc.Bytes()+ var probe map[string]json.RawMessage+ require.NoError(t, json.Unmarshal(line, &probe))+ if _, ok := probe["content"]; !ok {+ continue // action line+ }+ var doc copiedDoc+ require.NoError(t, json.Unmarshal(line, &doc))+ f.docs = append(f.docs, doc)+ }+ _, _ = w.Write([]byte(`{"took":1,"errors":false,"items":[]}`))+ default:+ t.Errorf("unexpected path: %s", r.URL.Path)+ }+ }+}++func TestCopyIndicesKeepsVectorsAndEnabledState(t *testing.T) {+ fake := &fakeCopyServer{}+ server := httptest.NewServer(fake.handler(t))+ defer server.Close()+ client, err := elasticsearch.NewClient(+ elasticsearch.Config{Addresses: []string{server.URL}, DisableRetry: true},+ )+ require.NoError(t, err)+ repo := &elasticsearchRepository{client: client, index: "vectors"}++ err = repo.CopyIndices(context.Background(), "kb-src",+ map[string]string{"k1": "tk1"},+ map[string]string{"c1": "t1", "c2": "t2", "c3": "t3"},+ "kb-dst", 2, "manual")+ require.NoError(t, err)++ assert.NotContains(t, fake.searchQuery, "is_enabled",+ "the copy scan must include disabled source rows so they stay disabled, not vanish")++ require.Len(t, fake.docs, 4)+ byContent := make(map[string]copiedDoc, len(fake.docs))+ for _, d := range fake.docs {+ byContent[d.Content] = d+ assert.Equal(t, "tk1", d.KnowledgeID)+ assert.Equal(t, "kb-dst", d.KnowledgeBaseID)+ require.NotNil(t, d.IsEnabled, "is_enabled must be written for %q", d.Content)+ }++ chunk := byContent["chunk"]+ assert.Equal(t, "t1", chunk.ChunkID)+ assert.Equal(t, "t1", chunk.SourceID)+ assert.Equal(t, 0, chunk.SourceType)+ assert.Equal(t, []float32{0.1, 0.2}, chunk.Embedding)+ assert.True(t, *chunk.IsEnabled)++ question := byContent["question"]+ assert.Equal(t, "t1", question.ChunkID)+ assert.Equal(t, "t1-q1", question.SourceID)+ assert.Equal(t, 1, question.SourceType)+ assert.Equal(t, []float32{0.3, 0.4}, question.Embedding)+ assert.True(t, *question.IsEnabled)++ summary := byContent["summary"]+ assert.Equal(t, "t2", summary.ChunkID)+ assert.NotEqual(t, "img-9", summary.SourceID)+ assert.False(t, strings.HasPrefix(summary.SourceID, "t2"), "unrelated source ids get a fresh uuid")+ assert.Equal(t, 2, summary.SourceType)+ assert.Equal(t, []float32{0.5, 0.6}, summary.Embedding)+ assert.False(t, *summary.IsEnabled)++ legacy := byContent["legacy"]+ assert.Equal(t, "t3", legacy.SourceID)+ assert.Equal(t, []float32{0.7, 0.8}, legacy.Embedding)+ assert.True(t, *legacy.IsEnabled, "a source row without is_enabled is enabled for retrieval")+}diff --git a/internal/application/repository/retriever/elasticsearch/v7/repository.go b/internal/application/repository/retriever/elasticsearch/v7/repository.goindex eee56bc26..973f9fb14 100644--- a/internal/application/repository/retriever/elasticsearch/v7/repository.go+++ b/internal/application/repository/retriever/elasticsearch/v7/repository.go@@ -997,11 +997,6 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,return nil}- // Build query parameters- retrieveParams := typesLocal.RetrieveParams{- KnowledgeBaseIDs: []string{sourceKnowledgeBaseID},- }-// Set batch processing parametersbatchSize := 500from := 0@@ -1009,7 +1004,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,for {// Query source data batch- hitsList, err := e.querySourceBatch(ctx, retrieveParams, from, batchSize)+ hitsList, err := e.querySourceBatch(ctx, sourceKnowledgeBaseID, from, batchSize)if err != nil {return err}@@ -1022,7 +1017,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,log.Infof("[ElasticsearchV7] Found %d source index data, batch start position: %d", len(hitsList), from)// Process the batch and create index information- indexInfoList, err := e.processSourceBatch(ctx, hitsList, sourceToTargetKBIDMap,+ indexInfoList, embeddingMap, err := e.processSourceBatch(ctx, hitsList, sourceToTargetKBIDMap,sourceToTargetChunkIDMap, targetKnowledgeBaseID)if err != nil {return err@@ -1030,7 +1025,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,// Save processed indicesif len(indexInfoList) > 0 {- err := e.saveCopiedIndices(ctx, indexInfoList)+ err := e.saveCopiedIndices(ctx, indexInfoList, embeddingMap)if err != nil {return err}@@ -1055,16 +1050,20 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,// querySourceBatch queries a batch of source datafunc (e *elasticsearchRepository) querySourceBatch(ctx context.Context,- retrieveParams typesLocal.RetrieveParams, from int, batchSize int,+ sourceKnowledgeBaseID string, from int, batchSize int,) ([]interface{}, error) {log := logger.GetLogger(ctx)- // Build query request safely- filterJSON := e.getBaseConds(retrieveParams)- var filter map[string]interface{}- if err := json.Unmarshal([]byte(filterJSON), &filter); err != nil {- log.Errorf("[ElasticsearchV7] Failed to parse base conditions: %v", err)- filter = map[string]interface{}{}+ // Scan every row of the source knowledge base. getBaseConds is not used+ // here because it drops disabled rows, which must be copied as disabled.+ filter := map[string]interface{}{+ "bool": map[string]interface{}{+ "filter": []map[string]interface{}{{+ "terms": map[string]interface{}{+ e.idField("knowledge_base_id"): []string{sourceKnowledgeBaseID},+ },+ }},+ },}queryBody := map[string]interface{}{@@ -1121,13 +1120,14 @@ func (e *elasticsearchRepository) querySourceBatch(ctx context.Context,return hitsList, nil}-// processSourceBatch processes a batch of source data and creates index information+// processSourceBatch processes a batch of source data and creates index information,+// returning the embeddings keyed by target SourceID (the key BatchSave looks up)func (e *elasticsearchRepository) processSourceBatch(ctx context.Context,hitsList []interface{},sourceToTargetKBIDMap map[string]string,sourceToTargetChunkIDMap map[string]string,targetKnowledgeBaseID string,-) ([]*typesLocal.IndexInfo, error) {+) ([]*typesLocal.IndexInfo, map[string][]float32, error) {log := logger.GetLogger(ctx)// Prepare index information for batch save@@ -1146,12 +1146,12 @@ func (e *elasticsearchRepository) processSourceBatch(ctx context.Context,if indexInfo != nil {indexInfoList = append(indexInfoList, indexInfo)if embeddingVector != nil {- embeddingMap[indexInfo.ChunkID] = embeddingVector+ embeddingMap[indexInfo.SourceID] = embeddingVector}}}- return indexInfoList, nil+ return indexInfoList, embeddingMap, nil}// processSingleHit processes a single hit and creates index information@@ -1271,7 +1271,9 @@ func (e *elasticsearchRepository) processSingleHit(ctx context.Context,}// saveCopiedIndices saves the copied indices-func (e *elasticsearchRepository) saveCopiedIndices(ctx context.Context, indexInfoList []*typesLocal.IndexInfo) error {+func (e *elasticsearchRepository) saveCopiedIndices(ctx context.Context,+ indexInfoList []*typesLocal.IndexInfo, embeddingMap map[string][]float32,+) error {log := logger.GetLogger(ctx)if len(indexInfoList) == 0 {@@ -1281,11 +1283,6 @@ func (e *elasticsearchRepository) saveCopiedIndices(ctx context.Context, indexIn// Prepare additional params with embedding mapadditionalParams := make(map[string]any)- embeddingMap := make(map[string][]float32)-- // No need to extract embeddings from metadata as they're not stored there- // We'll use the embeddings directly from the embedding map created in processSourceBatch-if len(embeddingMap) > 0 {additionalParams["embedding"] = embeddingMaplog.Infof("[ElasticsearchV7] Found %d embeddings to save", len(embeddingMap))diff --git a/internal/application/repository/retriever/elasticsearch/v8/copy_test.go b/internal/application/repository/retriever/elasticsearch/v8/copy_test.gonew file mode 100644index 000000000..794c90031--- /dev/null+++ b/internal/application/repository/retriever/elasticsearch/v8/copy_test.go@@ -0,0 +1,142 @@+package v8++import (+ "bufio"+ "bytes"+ "context"+ "encoding/json"+ "io"+ "net/http"+ "net/http/httptest"+ "strings"+ "sync"+ "testing"++ "github.com/elastic/go-elasticsearch/v8"+ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++// copySourceHits covers every source_id shape CopyIndices remaps: a regular+// chunk row, a generated-question row sharing that chunk, a row with an+// unrelated source_id (remapped to a fresh uuid), and a legacy row written+// before is_enabled existed.+const copySourceHits = `[+ {"_index":"vectors","_id":"d1","_score":1,"_source":{"content":"chunk","source_id":"c1","source_type":0,+ "chunk_id":"c1","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.1,0.2],"is_enabled":true}},+ {"_index":"vectors","_id":"d2","_score":1,"_source":{"content":"question","source_id":"c1-q1","source_type":1,+ "chunk_id":"c1","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.3,0.4],"is_enabled":true}},+ {"_index":"vectors","_id":"d3","_score":1,"_source":{"content":"summary","source_id":"img-9","source_type":2,+ "chunk_id":"c2","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.5,0.6],"is_enabled":false}},+ {"_index":"vectors","_id":"d4","_score":1,"_source":{"content":"legacy","source_id":"c3","source_type":0,+ "chunk_id":"c3","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.7,0.8]}}+]`++type copiedDoc struct {+ Content string `json:"content"`+ SourceID string `json:"source_id"`+ SourceType int `json:"source_type"`+ ChunkID string `json:"chunk_id"`+ KnowledgeID string `json:"knowledge_id"`+ KnowledgeBaseID string `json:"knowledge_base_id"`+ Embedding []float32 `json:"embedding"`+ IsEnabled *bool `json:"is_enabled"`+}++// fakeCopyServer serves one page of copySourceHits and records the search+// query and every document sent through _bulk.+type fakeCopyServer struct {+ mu sync.Mutex+ searchQuery string+ docs []copiedDoc+}++func (f *fakeCopyServer) handler(t *testing.T) http.HandlerFunc {+ return func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ w.Header().Set("X-Elastic-Product", "Elasticsearch")+ body, _ := io.ReadAll(r.Body)+ f.mu.Lock()+ defer f.mu.Unlock()+ switch r.URL.Path {+ case "/vectors/_search":+ f.searchQuery = string(body)+ _, _ = w.Write([]byte(`{"took":1,"timed_out":false,` ++ `"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},` ++ `"hits":{"total":{"value":4,"relation":"eq"},"hits":` + copySourceHits + `}}`))+ case "/vectors/_bulk":+ sc := bufio.NewScanner(bytes.NewReader(body))+ sc.Buffer(make([]byte, 0, 1<<20), 1<<20)+ for sc.Scan() {+ line := sc.Bytes()+ var probe map[string]json.RawMessage+ require.NoError(t, json.Unmarshal(line, &probe))+ if _, ok := probe["content"]; !ok {+ continue // action line+ }+ var doc copiedDoc+ require.NoError(t, json.Unmarshal(line, &doc))+ f.docs = append(f.docs, doc)+ }+ _, _ = w.Write([]byte(`{"took":1,"errors":false,"items":[]}`))+ default:+ t.Errorf("unexpected path: %s", r.URL.Path)+ }+ }+}++func TestCopyIndicesKeepsVectorsAndEnabledState(t *testing.T) {+ fake := &fakeCopyServer{}+ server := httptest.NewServer(fake.handler(t))+ defer server.Close()+ client, err := elasticsearch.NewTypedClient(+ elasticsearch.Config{Addresses: []string{server.URL}, DisableRetry: true},+ )+ require.NoError(t, err)+ repo := &elasticsearchRepository{client: client, index: "vectors"}++ err = repo.CopyIndices(context.Background(), "kb-src",+ map[string]string{"k1": "tk1"},+ map[string]string{"c1": "t1", "c2": "t2", "c3": "t3"},+ "kb-dst", 2, "manual")+ require.NoError(t, err)++ assert.NotContains(t, fake.searchQuery, "is_enabled",+ "the copy scan must include disabled source rows so they stay disabled, not vanish")++ require.Len(t, fake.docs, 4)+ byContent := make(map[string]copiedDoc, len(fake.docs))+ for _, d := range fake.docs {+ byContent[d.Content] = d+ assert.Equal(t, "tk1", d.KnowledgeID)+ assert.Equal(t, "kb-dst", d.KnowledgeBaseID)+ require.NotNil(t, d.IsEnabled, "is_enabled must be written for %q", d.Content)+ }++ chunk := byContent["chunk"]+ assert.Equal(t, "t1", chunk.ChunkID)+ assert.Equal(t, "t1", chunk.SourceID)+ assert.Equal(t, 0, chunk.SourceType)+ assert.Equal(t, []float32{0.1, 0.2}, chunk.Embedding)+ assert.True(t, *chunk.IsEnabled)++ question := byContent["question"]+ assert.Equal(t, "t1", question.ChunkID)+ assert.Equal(t, "t1-q1", question.SourceID)+ assert.Equal(t, 1, question.SourceType)+ assert.Equal(t, []float32{0.3, 0.4}, question.Embedding)+ assert.True(t, *question.IsEnabled)++ summary := byContent["summary"]+ assert.Equal(t, "t2", summary.ChunkID)+ assert.NotEqual(t, "img-9", summary.SourceID)+ assert.False(t, strings.HasPrefix(summary.SourceID, "t2"), "unrelated source ids get a fresh uuid")+ assert.Equal(t, 2, summary.SourceType)+ assert.Equal(t, []float32{0.5, 0.6}, summary.Embedding)+ assert.False(t, *summary.IsEnabled)++ legacy := byContent["legacy"]+ assert.Equal(t, "t3", legacy.SourceID)+ assert.Equal(t, []float32{0.7, 0.8}, legacy.Embedding)+ assert.True(t, *legacy.IsEnabled, "a source row without is_enabled is enabled for retrieval")+}diff --git a/internal/application/repository/retriever/elasticsearch/v8/repository.go b/internal/application/repository/retriever/elasticsearch/v8/repository.goindex 63134307b..9b10671d1 100644--- a/internal/application/repository/retriever/elasticsearch/v8/repository.go+++ b/internal/application/repository/retriever/elasticsearch/v8/repository.go@@ -557,6 +557,14 @@ func (e *elasticsearchRepository) KeywordsRetrieve(ctx context.Context,}, nil}+// copySourceDoc decodes a source row for CopyIndices. IsEnabled shadows the+// embedded field so a row written before is_enabled existed, which retrieval+// treats as enabled, can be told apart from an explicit false.+type copySourceDoc struct {+ elasticsearchRetriever.VectorEmbedding+ IsEnabled *bool `json:"is_enabled"`+}+// CopyIndices 复制索引数据func (e *elasticsearchRepository) CopyIndices(ctx context.Context,sourceKnowledgeBaseID string,@@ -577,13 +585,13 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,return nil}- // Build query parameters- params := typesLocal.RetrieveParams{- KnowledgeBaseIDs: []string{sourceKnowledgeBaseID},- }-- // Build base query conditions- filter := e.getBaseConds(params)+ // Scan every row of the source knowledge base. getBaseConds is not used+ // here because it drops disabled rows, which must be copied as disabled.+ filter := []types.Query{{Terms: &types.TermsQuery{+ TermsQuery: map[string]types.TermsQueryField{+ e.idField("knowledge_base_id"): []string{sourceKnowledgeBaseID},+ },+ }}}// Set batch processing parametersbatchSize := 500@@ -617,7 +625,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,for _, hit := range searchResponse.Hits.Hits {// Parse source document- var sourceDoc elasticsearchRetriever.VectorEmbedding+ var sourceDoc copySourceDocif err := json.Unmarshal(hit.Source_, &sourceDoc); err != nil {log.Errorf("[Elasticsearch] Failed to parse source index data: %v", err)continue@@ -639,11 +647,6 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,continue}- // Save embedding vector to embeddingMap- if len(sourceDoc.Embedding) > 0 {- embeddingMap[targetChunkID] = sourceDoc.Embedding- }-// Handle SourceID transformation for generated questions// Generated questions have SourceID format: {chunkID}-{questionID}// Regular chunks have SourceID == ChunkID@@ -660,6 +663,12 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,targetSourceID = uuid.New().String()}+ // BatchSave looks embeddings up by SourceID, so key by the target+ // SourceID; rows of one chunk would collide on the chunk ID.+ if len(sourceDoc.Embedding) > 0 {+ embeddingMap[targetSourceID] = sourceDoc.Embedding+ }+// Create new index informationindexInfo := &typesLocal.IndexInfo{Content: sourceDoc.Content,@@ -668,6 +677,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,ChunkID: targetChunkID,KnowledgeID: targetKnowledgeID,KnowledgeBaseID: targetKnowledgeBaseID,+ IsEnabled: sourceDoc.IsEnabled == nil || *sourceDoc.IsEnabled,}indexInfoList = append(indexInfoList, indexInfo)