Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

[Go] Fix switch embeding report error (#19201)

Wang Qiae556538ac3c
diff --git a/internal/service/dataset/embedding.go b/internal/service/dataset/embedding.go
index 6149976640..b654dcb3fe 100644
--- a/internal/service/dataset/embedding.go
+++ b/internal/service/dataset/embedding.go
@@ -49,7 +49,7 @@ func (d *DatasetService) CheckEmbedding(ctx context.Context, userID, datasetID s
if dao.IsNotFoundErr(err) {
return nil, common.CodeDataError, errors.New("invalid Dataset ID")
}
- return nil, common.CodeServerError, errors.New("internal server error")
+ return nil, common.CodeServerError, fmt.Errorf("failed to get dataset %s: %w", datasetID, err)
}
if req == nil || strings.TrimSpace(req.EmbeddingID) == "" {
@@ -95,11 +95,11 @@ func (d *DatasetService) CheckEmbedding(ctx context.Context, userID, datasetID s
rawChunk, err := d.docEngine.GetChunk(ctx, fmt.Sprintf("ragflow_%s", kb.TenantID), sample.ChunkID, []string{datasetID})
if err != nil {
- continue
+ return nil, common.CodeServerError, fmt.Errorf("failed to get sampled chunk %s: %w", sample.ChunkID, err)
}
chunkMap := datasetMap(rawChunk)
if len(chunkMap) == 0 {
- continue
+ return nil, common.CodeServerError, fmt.Errorf("sampled chunk %s was not found", sample.ChunkID)
}
title := datasetString(chunkMap["title_tks"])
@@ -191,7 +191,7 @@ func (d *DatasetService) sampleRandomChunksWithVectors(ctx context.Context, tena
},
})
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to count chunks for dataset %s: %w", datasetID, err)
}
if totalResult == nil || totalResult.Total <= 0 {
return []embeddingCheckSample{}, nil
@@ -223,6 +223,7 @@ func (d *DatasetService) sampleRandomChunksWithVectors(ctx context.Context, tena
baseFields := []string{"docnm_kwd", "doc_id", "content_with_weight", "page_num_int", "position_int", "top_int"}
samples := make([]embeddingCheckSample, 0, n)
+ var lastSampleErr error
for _, offset := range offsets {
searchResult, err := d.docEngine.Search(ctx, &enginetypes.SearchRequest{
IndexNames: []string{indexName},
@@ -236,21 +237,24 @@ func (d *DatasetService) sampleRandomChunksWithVectors(ctx context.Context, tena
},
})
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to sample chunk at offset %d for dataset %s: %w", offset, datasetID, err)
}
if searchResult == nil || len(searchResult.Chunks) == 0 {
+ lastSampleErr = fmt.Errorf("chunk search returned no result at offset %d for dataset %s", offset, datasetID)
continue
}
chunkID := datasetChunkID(searchResult.Chunks[0])
if chunkID == "" {
+ lastSampleErr = fmt.Errorf("sampled chunk at offset %d for dataset %s is missing an ID", offset, datasetID)
continue
}
fullChunk, err := d.docEngine.GetChunk(ctx, indexName, chunkID, []string{datasetID})
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("failed to get sampled chunk %s: %w", chunkID, err)
}
chunkMap := datasetMap(fullChunk)
if len(chunkMap) == 0 {
+ lastSampleErr = fmt.Errorf("sampled chunk %s was not found in dataset %s", chunkID, datasetID)
continue
}
vectorField := datasetGuessVecField(chunkMap)
@@ -266,12 +270,12 @@ func (d *DatasetService) sampleRandomChunksWithVectors(ctx context.Context, tena
Position: chunkMap["position_int"],
Top: chunkMap["top_int"],
ContentWithWeight: datasetString(chunkMap["content_with_weight"]),
- QuestionKeywords: datasetStringSlice(chunkMap["question_keywords"]),
+ QuestionKeywords: datasetStringSlice(chunkMap["question_kwd"]),
})
}
if len(samples) == 0 {
- return nil, errors.New("no valid chunks with vectors found")
+ return nil, lastSampleErr
}
return samples, nil
}
diff --git a/internal/service/dataset/utils.go b/internal/service/dataset/utils.go
index 7c251b7c44..204a39eddc 100644
--- a/internal/service/dataset/utils.go
+++ b/internal/service/dataset/utils.go
@@ -6,6 +6,7 @@ import (
"math"
"regexp"
"sort"
+ "strconv"
"strings"
"time"
@@ -163,45 +164,64 @@ func datasetStringSlice(value interface{}) []string {
}
func datasetGuessVecField(src map[string]interface{}) string {
- var f64, f32 string
- for k, v := range src {
- if !strings.HasPrefix(k, "q_") && !strings.HasPrefix(k, "u_") {
- continue
+ for k := range src {
+ if strings.HasSuffix(k, "_vec") {
+ return k
}
- switch v.(type) {
- case []float64:
- f64 = k
- case string:
- f32 = k
- }
- }
- if f64 != "" {
- return f64
}
- return f32
+ return ""
}
func datasetAsFloatVec(v interface{}) []float64 {
switch val := v.(type) {
case []float64:
return val
+ case []float32:
+ vec := make([]float64, len(val))
+ for i, n := range val {
+ vec[i] = float64(n)
+ }
+ return vec
case []interface{}:
vec := make([]float64, 0, len(val))
for _, item := range val {
switch n := item.(type) {
case float64:
vec = append(vec, n)
+ case float32:
+ vec = append(vec, float64(n))
case int:
vec = append(vec, float64(n))
case int64:
vec = append(vec, float64(n))
case json.Number:
- if f, err := n.Float64(); err == nil {
- vec = append(vec, f)
+ f, err := n.Float64()
+ if err != nil {
+ return nil
+ }
+ vec = append(vec, f)
+ case string:
+ f, err := strconv.ParseFloat(strings.TrimSpace(n), 64)
+ if err != nil {
+ return nil
}
+ vec = append(vec, f)
+ default:
+ return nil
}
}
return vec
+ case string:
+ parts := strings.FieldsFunc(strings.Trim(val, "[]{}"), func(r rune) bool { return r == '\t' || r == ',' })
+ vec := make([]float64, 0, len(parts))
+ for _, part := range parts {
+ f, err := strconv.ParseFloat(strings.TrimSpace(part), 64)
+ if err != nil {
+ return nil
+ }
+ vec = append(vec, f)
+ }
+ return vec
}
return nil
}
@@ -309,6 +329,11 @@ func datasetChunkID(chunk map[string]interface{}) string {
return s
}
}
+ if id, ok := chunk["id"]; ok {
+ if s, ok := id.(string); ok {
+ return s
+ }
+ }
return ""
}