Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(retriever/es): keep vectors and enabled state on CopyIndices (#3678)

lyingbug65d8129cb64f
diff --git a/internal/application/repository/retriever/elasticsearch/v7/copy_test.go b/internal/application/repository/retriever/elasticsearch/v7/copy_test.go
new file mode 100644
index 000000000..1cc4aa1af
--- /dev/null
+++ b/internal/application/repository/retriever/elasticsearch/v7/copy_test.go
@@ -0,0 +1,145 @@
+package v7
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/elastic/go-elasticsearch/v7"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// copySourceHits covers every source_id shape CopyIndices remaps: a regular
+// chunk row, a generated-question row sharing that chunk, a row with an
+// unrelated source_id (remapped to a fresh uuid), and a legacy row written
+// before is_enabled existed.
+const copySourceHits = `[
+ {"_index":"vectors","_id":"d1","_score":1,"_source":{"content":"chunk","source_id":"c1","source_type":0,
+ "chunk_id":"c1","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.1,0.2],"is_enabled":true}},
+ {"_index":"vectors","_id":"d2","_score":1,"_source":{"content":"question","source_id":"c1-q1","source_type":1,
+ "chunk_id":"c1","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.3,0.4],"is_enabled":true}},
+ {"_index":"vectors","_id":"d3","_score":1,"_source":{"content":"summary","source_id":"img-9","source_type":2,
+ "chunk_id":"c2","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.5,0.6],"is_enabled":false}},
+ {"_index":"vectors","_id":"d4","_score":1,"_source":{"content":"legacy","source_id":"c3","source_type":0,
+ "chunk_id":"c3","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.7,0.8]}}
+]`
+
+type copiedDoc struct {
+ Content string `json:"content"`
+ SourceID string `json:"source_id"`
+ SourceType int `json:"source_type"`
+ ChunkID string `json:"chunk_id"`
+ KnowledgeID string `json:"knowledge_id"`
+ KnowledgeBaseID string `json:"knowledge_base_id"`
+ Embedding []float32 `json:"embedding"`
+ IsEnabled *bool `json:"is_enabled"`
+}
+
+// fakeCopyServer serves one page of copySourceHits and records the search
+// query and every document sent through _bulk.
+type fakeCopyServer struct {
+ mu sync.Mutex
+ searchQuery string
+ docs []copiedDoc
+}
+
+func (f *fakeCopyServer) handler(t *testing.T) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ body, _ := io.ReadAll(r.Body)
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ switch r.URL.Path {
+ case "/":
+ _, _ = w.Write([]byte(`{"version":{"number":"7.17.0","build_flavor":"default"},` +
+ `"tagline":"You Know, for Search"}`))
+ case "/vectors/_search":
+ f.searchQuery = string(body)
+ _, _ = w.Write([]byte(`{"took":1,"timed_out":false,` +
+ `"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},` +
+ `"hits":{"total":{"value":4,"relation":"eq"},"hits":` + copySourceHits + `}}`))
+ case "/vectors/_bulk":
+ sc := bufio.NewScanner(bytes.NewReader(body))
+ sc.Buffer(make([]byte, 0, 1<<20), 1<<20)
+ for sc.Scan() {
+ line := sc.Bytes()
+ var probe map[string]json.RawMessage
+ require.NoError(t, json.Unmarshal(line, &probe))
+ if _, ok := probe["content"]; !ok {
+ continue // action line
+ }
+ var doc copiedDoc
+ require.NoError(t, json.Unmarshal(line, &doc))
+ f.docs = append(f.docs, doc)
+ }
+ _, _ = w.Write([]byte(`{"took":1,"errors":false,"items":[]}`))
+ default:
+ t.Errorf("unexpected path: %s", r.URL.Path)
+ }
+ }
+}
+
+func TestCopyIndicesKeepsVectorsAndEnabledState(t *testing.T) {
+ fake := &fakeCopyServer{}
+ server := httptest.NewServer(fake.handler(t))
+ defer server.Close()
+ client, err := elasticsearch.NewClient(
+ elasticsearch.Config{Addresses: []string{server.URL}, DisableRetry: true},
+ )
+ require.NoError(t, err)
+ repo := &elasticsearchRepository{client: client, index: "vectors"}
+
+ err = repo.CopyIndices(context.Background(), "kb-src",
+ map[string]string{"k1": "tk1"},
+ map[string]string{"c1": "t1", "c2": "t2", "c3": "t3"},
+ "kb-dst", 2, "manual")
+ require.NoError(t, err)
+
+ assert.NotContains(t, fake.searchQuery, "is_enabled",
+ "the copy scan must include disabled source rows so they stay disabled, not vanish")
+
+ require.Len(t, fake.docs, 4)
+ byContent := make(map[string]copiedDoc, len(fake.docs))
+ for _, d := range fake.docs {
+ byContent[d.Content] = d
+ assert.Equal(t, "tk1", d.KnowledgeID)
+ assert.Equal(t, "kb-dst", d.KnowledgeBaseID)
+ require.NotNil(t, d.IsEnabled, "is_enabled must be written for %q", d.Content)
+ }
+
+ chunk := byContent["chunk"]
+ assert.Equal(t, "t1", chunk.ChunkID)
+ assert.Equal(t, "t1", chunk.SourceID)
+ assert.Equal(t, 0, chunk.SourceType)
+ assert.Equal(t, []float32{0.1, 0.2}, chunk.Embedding)
+ assert.True(t, *chunk.IsEnabled)
+
+ question := byContent["question"]
+ assert.Equal(t, "t1", question.ChunkID)
+ assert.Equal(t, "t1-q1", question.SourceID)
+ assert.Equal(t, 1, question.SourceType)
+ assert.Equal(t, []float32{0.3, 0.4}, question.Embedding)
+ assert.True(t, *question.IsEnabled)
+
+ summary := byContent["summary"]
+ assert.Equal(t, "t2", summary.ChunkID)
+ assert.NotEqual(t, "img-9", summary.SourceID)
+ assert.False(t, strings.HasPrefix(summary.SourceID, "t2"), "unrelated source ids get a fresh uuid")
+ assert.Equal(t, 2, summary.SourceType)
+ assert.Equal(t, []float32{0.5, 0.6}, summary.Embedding)
+ assert.False(t, *summary.IsEnabled)
+
+ legacy := byContent["legacy"]
+ assert.Equal(t, "t3", legacy.SourceID)
+ assert.Equal(t, []float32{0.7, 0.8}, legacy.Embedding)
+ assert.True(t, *legacy.IsEnabled, "a source row without is_enabled is enabled for retrieval")
+}
diff --git a/internal/application/repository/retriever/elasticsearch/v7/repository.go b/internal/application/repository/retriever/elasticsearch/v7/repository.go
index eee56bc26..973f9fb14 100644
--- a/internal/application/repository/retriever/elasticsearch/v7/repository.go
+++ b/internal/application/repository/retriever/elasticsearch/v7/repository.go
@@ -997,11 +997,6 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
return nil
}
- // Build query parameters
- retrieveParams := typesLocal.RetrieveParams{
- KnowledgeBaseIDs: []string{sourceKnowledgeBaseID},
- }
-
// Set batch processing parameters
batchSize := 500
from := 0
@@ -1009,7 +1004,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
for {
// Query source data batch
- hitsList, err := e.querySourceBatch(ctx, retrieveParams, from, batchSize)
+ hitsList, err := e.querySourceBatch(ctx, sourceKnowledgeBaseID, from, batchSize)
if err != nil {
return err
}
@@ -1022,7 +1017,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
log.Infof("[ElasticsearchV7] Found %d source index data, batch start position: %d", len(hitsList), from)
// Process the batch and create index information
- indexInfoList, err := e.processSourceBatch(ctx, hitsList, sourceToTargetKBIDMap,
+ indexInfoList, embeddingMap, err := e.processSourceBatch(ctx, hitsList, sourceToTargetKBIDMap,
sourceToTargetChunkIDMap, targetKnowledgeBaseID)
if err != nil {
return err
@@ -1030,7 +1025,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
// Save processed indices
if len(indexInfoList) > 0 {
- err := e.saveCopiedIndices(ctx, indexInfoList)
+ err := e.saveCopiedIndices(ctx, indexInfoList, embeddingMap)
if err != nil {
return err
}
@@ -1055,16 +1050,20 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
// querySourceBatch queries a batch of source data
func (e *elasticsearchRepository) querySourceBatch(ctx context.Context,
- retrieveParams typesLocal.RetrieveParams, from int, batchSize int,
+ sourceKnowledgeBaseID string, from int, batchSize int,
) ([]interface{}, error) {
log := logger.GetLogger(ctx)
- // Build query request safely
- filterJSON := e.getBaseConds(retrieveParams)
- var filter map[string]interface{}
- if err := json.Unmarshal([]byte(filterJSON), &filter); err != nil {
- log.Errorf("[ElasticsearchV7] Failed to parse base conditions: %v", err)
- filter = map[string]interface{}{}
+ // Scan every row of the source knowledge base. getBaseConds is not used
+ // here because it drops disabled rows, which must be copied as disabled.
+ filter := map[string]interface{}{
+ "bool": map[string]interface{}{
+ "filter": []map[string]interface{}{{
+ "terms": map[string]interface{}{
+ e.idField("knowledge_base_id"): []string{sourceKnowledgeBaseID},
+ },
+ }},
+ },
}
queryBody := map[string]interface{}{
@@ -1121,13 +1120,14 @@ func (e *elasticsearchRepository) querySourceBatch(ctx context.Context,
return hitsList, nil
}
-// processSourceBatch processes a batch of source data and creates index information
+// processSourceBatch processes a batch of source data and creates index information,
+// returning the embeddings keyed by target SourceID (the key BatchSave looks up)
func (e *elasticsearchRepository) processSourceBatch(ctx context.Context,
hitsList []interface{},
sourceToTargetKBIDMap map[string]string,
sourceToTargetChunkIDMap map[string]string,
targetKnowledgeBaseID string,
-) ([]*typesLocal.IndexInfo, error) {
+) ([]*typesLocal.IndexInfo, map[string][]float32, error) {
log := logger.GetLogger(ctx)
// Prepare index information for batch save
@@ -1146,12 +1146,12 @@ func (e *elasticsearchRepository) processSourceBatch(ctx context.Context,
if indexInfo != nil {
indexInfoList = append(indexInfoList, indexInfo)
if embeddingVector != nil {
- embeddingMap[indexInfo.ChunkID] = embeddingVector
+ embeddingMap[indexInfo.SourceID] = embeddingVector
}
}
}
- return indexInfoList, nil
+ return indexInfoList, embeddingMap, nil
}
// processSingleHit processes a single hit and creates index information
@@ -1271,7 +1271,9 @@ func (e *elasticsearchRepository) processSingleHit(ctx context.Context,
}
// saveCopiedIndices saves the copied indices
-func (e *elasticsearchRepository) saveCopiedIndices(ctx context.Context, indexInfoList []*typesLocal.IndexInfo) error {
+func (e *elasticsearchRepository) saveCopiedIndices(ctx context.Context,
+ indexInfoList []*typesLocal.IndexInfo, embeddingMap map[string][]float32,
+) error {
log := logger.GetLogger(ctx)
if len(indexInfoList) == 0 {
@@ -1281,11 +1283,6 @@ func (e *elasticsearchRepository) saveCopiedIndices(ctx context.Context, indexIn
// Prepare additional params with embedding map
additionalParams := make(map[string]any)
- embeddingMap := make(map[string][]float32)
-
- // No need to extract embeddings from metadata as they're not stored there
- // We'll use the embeddings directly from the embedding map created in processSourceBatch
-
if len(embeddingMap) > 0 {
additionalParams["embedding"] = embeddingMap
log.Infof("[ElasticsearchV7] Found %d embeddings to save", len(embeddingMap))
diff --git a/internal/application/repository/retriever/elasticsearch/v8/copy_test.go b/internal/application/repository/retriever/elasticsearch/v8/copy_test.go
new file mode 100644
index 000000000..794c90031
--- /dev/null
+++ b/internal/application/repository/retriever/elasticsearch/v8/copy_test.go
@@ -0,0 +1,142 @@
+package v8
+
+import (
+ "bufio"
+ "bytes"
+ "context"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/elastic/go-elasticsearch/v8"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+// copySourceHits covers every source_id shape CopyIndices remaps: a regular
+// chunk row, a generated-question row sharing that chunk, a row with an
+// unrelated source_id (remapped to a fresh uuid), and a legacy row written
+// before is_enabled existed.
+const copySourceHits = `[
+ {"_index":"vectors","_id":"d1","_score":1,"_source":{"content":"chunk","source_id":"c1","source_type":0,
+ "chunk_id":"c1","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.1,0.2],"is_enabled":true}},
+ {"_index":"vectors","_id":"d2","_score":1,"_source":{"content":"question","source_id":"c1-q1","source_type":1,
+ "chunk_id":"c1","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.3,0.4],"is_enabled":true}},
+ {"_index":"vectors","_id":"d3","_score":1,"_source":{"content":"summary","source_id":"img-9","source_type":2,
+ "chunk_id":"c2","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.5,0.6],"is_enabled":false}},
+ {"_index":"vectors","_id":"d4","_score":1,"_source":{"content":"legacy","source_id":"c3","source_type":0,
+ "chunk_id":"c3","knowledge_id":"k1","knowledge_base_id":"kb-src","embedding":[0.7,0.8]}}
+]`
+
+type copiedDoc struct {
+ Content string `json:"content"`
+ SourceID string `json:"source_id"`
+ SourceType int `json:"source_type"`
+ ChunkID string `json:"chunk_id"`
+ KnowledgeID string `json:"knowledge_id"`
+ KnowledgeBaseID string `json:"knowledge_base_id"`
+ Embedding []float32 `json:"embedding"`
+ IsEnabled *bool `json:"is_enabled"`
+}
+
+// fakeCopyServer serves one page of copySourceHits and records the search
+// query and every document sent through _bulk.
+type fakeCopyServer struct {
+ mu sync.Mutex
+ searchQuery string
+ docs []copiedDoc
+}
+
+func (f *fakeCopyServer) handler(t *testing.T) http.HandlerFunc {
+ return func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ body, _ := io.ReadAll(r.Body)
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ switch r.URL.Path {
+ case "/vectors/_search":
+ f.searchQuery = string(body)
+ _, _ = w.Write([]byte(`{"took":1,"timed_out":false,` +
+ `"_shards":{"total":1,"successful":1,"skipped":0,"failed":0},` +
+ `"hits":{"total":{"value":4,"relation":"eq"},"hits":` + copySourceHits + `}}`))
+ case "/vectors/_bulk":
+ sc := bufio.NewScanner(bytes.NewReader(body))
+ sc.Buffer(make([]byte, 0, 1<<20), 1<<20)
+ for sc.Scan() {
+ line := sc.Bytes()
+ var probe map[string]json.RawMessage
+ require.NoError(t, json.Unmarshal(line, &probe))
+ if _, ok := probe["content"]; !ok {
+ continue // action line
+ }
+ var doc copiedDoc
+ require.NoError(t, json.Unmarshal(line, &doc))
+ f.docs = append(f.docs, doc)
+ }
+ _, _ = w.Write([]byte(`{"took":1,"errors":false,"items":[]}`))
+ default:
+ t.Errorf("unexpected path: %s", r.URL.Path)
+ }
+ }
+}
+
+func TestCopyIndicesKeepsVectorsAndEnabledState(t *testing.T) {
+ fake := &fakeCopyServer{}
+ server := httptest.NewServer(fake.handler(t))
+ defer server.Close()
+ client, err := elasticsearch.NewTypedClient(
+ elasticsearch.Config{Addresses: []string{server.URL}, DisableRetry: true},
+ )
+ require.NoError(t, err)
+ repo := &elasticsearchRepository{client: client, index: "vectors"}
+
+ err = repo.CopyIndices(context.Background(), "kb-src",
+ map[string]string{"k1": "tk1"},
+ map[string]string{"c1": "t1", "c2": "t2", "c3": "t3"},
+ "kb-dst", 2, "manual")
+ require.NoError(t, err)
+
+ assert.NotContains(t, fake.searchQuery, "is_enabled",
+ "the copy scan must include disabled source rows so they stay disabled, not vanish")
+
+ require.Len(t, fake.docs, 4)
+ byContent := make(map[string]copiedDoc, len(fake.docs))
+ for _, d := range fake.docs {
+ byContent[d.Content] = d
+ assert.Equal(t, "tk1", d.KnowledgeID)
+ assert.Equal(t, "kb-dst", d.KnowledgeBaseID)
+ require.NotNil(t, d.IsEnabled, "is_enabled must be written for %q", d.Content)
+ }
+
+ chunk := byContent["chunk"]
+ assert.Equal(t, "t1", chunk.ChunkID)
+ assert.Equal(t, "t1", chunk.SourceID)
+ assert.Equal(t, 0, chunk.SourceType)
+ assert.Equal(t, []float32{0.1, 0.2}, chunk.Embedding)
+ assert.True(t, *chunk.IsEnabled)
+
+ question := byContent["question"]
+ assert.Equal(t, "t1", question.ChunkID)
+ assert.Equal(t, "t1-q1", question.SourceID)
+ assert.Equal(t, 1, question.SourceType)
+ assert.Equal(t, []float32{0.3, 0.4}, question.Embedding)
+ assert.True(t, *question.IsEnabled)
+
+ summary := byContent["summary"]
+ assert.Equal(t, "t2", summary.ChunkID)
+ assert.NotEqual(t, "img-9", summary.SourceID)
+ assert.False(t, strings.HasPrefix(summary.SourceID, "t2"), "unrelated source ids get a fresh uuid")
+ assert.Equal(t, 2, summary.SourceType)
+ assert.Equal(t, []float32{0.5, 0.6}, summary.Embedding)
+ assert.False(t, *summary.IsEnabled)
+
+ legacy := byContent["legacy"]
+ assert.Equal(t, "t3", legacy.SourceID)
+ assert.Equal(t, []float32{0.7, 0.8}, legacy.Embedding)
+ assert.True(t, *legacy.IsEnabled, "a source row without is_enabled is enabled for retrieval")
+}
diff --git a/internal/application/repository/retriever/elasticsearch/v8/repository.go b/internal/application/repository/retriever/elasticsearch/v8/repository.go
index 63134307b..9b10671d1 100644
--- a/internal/application/repository/retriever/elasticsearch/v8/repository.go
+++ b/internal/application/repository/retriever/elasticsearch/v8/repository.go
@@ -557,6 +557,14 @@ func (e *elasticsearchRepository) KeywordsRetrieve(ctx context.Context,
}, nil
}
+// copySourceDoc decodes a source row for CopyIndices. IsEnabled shadows the
+// embedded field so a row written before is_enabled existed, which retrieval
+// treats as enabled, can be told apart from an explicit false.
+type copySourceDoc struct {
+ elasticsearchRetriever.VectorEmbedding
+ IsEnabled *bool `json:"is_enabled"`
+}
+
// CopyIndices 复制索引数据
func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
sourceKnowledgeBaseID string,
@@ -577,13 +585,13 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
return nil
}
- // Build query parameters
- params := typesLocal.RetrieveParams{
- KnowledgeBaseIDs: []string{sourceKnowledgeBaseID},
- }
-
- // Build base query conditions
- filter := e.getBaseConds(params)
+ // Scan every row of the source knowledge base. getBaseConds is not used
+ // here because it drops disabled rows, which must be copied as disabled.
+ filter := []types.Query{{Terms: &types.TermsQuery{
+ TermsQuery: map[string]types.TermsQueryField{
+ e.idField("knowledge_base_id"): []string{sourceKnowledgeBaseID},
+ },
+ }}}
// Set batch processing parameters
batchSize := 500
@@ -617,7 +625,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
for _, hit := range searchResponse.Hits.Hits {
// Parse source document
- var sourceDoc elasticsearchRetriever.VectorEmbedding
+ var sourceDoc copySourceDoc
if err := json.Unmarshal(hit.Source_, &sourceDoc); err != nil {
log.Errorf("[Elasticsearch] Failed to parse source index data: %v", err)
continue
@@ -639,11 +647,6 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
continue
}
- // Save embedding vector to embeddingMap
- if len(sourceDoc.Embedding) > 0 {
- embeddingMap[targetChunkID] = sourceDoc.Embedding
- }
-
// Handle SourceID transformation for generated questions
// Generated questions have SourceID format: {chunkID}-{questionID}
// Regular chunks have SourceID == ChunkID
@@ -660,6 +663,12 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
targetSourceID = uuid.New().String()
}
+ // BatchSave looks embeddings up by SourceID, so key by the target
+ // SourceID; rows of one chunk would collide on the chunk ID.
+ if len(sourceDoc.Embedding) > 0 {
+ embeddingMap[targetSourceID] = sourceDoc.Embedding
+ }
+
// Create new index information
indexInfo := &typesLocal.IndexInfo{
Content: sourceDoc.Content,
@@ -668,6 +677,7 @@ func (e *elasticsearchRepository) CopyIndices(ctx context.Context,
ChunkID: targetChunkID,
KnowledgeID: targetKnowledgeID,
KnowledgeBaseID: targetKnowledgeBaseID,
+ IsEnabled: sourceDoc.IsEnabled == nil || *sourceDoc.IsEnabled,
}
indexInfoList = append(indexInfoList, indexInfo)