Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

46.2 KB
Raw Blame
package elasticsearch

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"reflect"
	"strings"
	"sync/atomic"
	"testing"

	"github.com/elastic/go-elasticsearch/v8"

	"ragflow/internal/common"
	"ragflow/internal/engine/types"
)

func TestBuildQueryStringQueryMapsSkillFieldsToTokenFields(t *testing.T) {
	query := buildQueryStringQuery(&types.MatchTextExpr{
		MatchingText: "test",
		Fields:       []string{"name^10", "tags^5", "description^3", "content^1"},
	}, true, false)

	queryString, ok := query["query_string"].(map[string]interface{})
	if !ok {
		t.Fatalf("query_string missing from %#v", query)
	}
	assertEqual(t, queryString["fields"], []string{"name_tks^10", "tags_tks^5", "description_tks^3", "content_tks^1"})
	assertEqual(t, queryString["query"], "test")
}

func TestBuildQueryStringQueryKeepsDocumentFieldsUnchanged(t *testing.T) {
	query := buildQueryStringQuery(&types.MatchTextExpr{
		MatchingText: "test",
		Fields:       []string{"name^10"},
	}, false, false)

	queryString, ok := query["query_string"].(map[string]interface{})
	if !ok {
		t.Fatalf("query_string missing from %#v", query)
	}
	assertEqual(t, queryString["fields"], []string{"name^10"})
}

func TestBuildQueryStringQueryLowercasesMatchingText(t *testing.T) {
	// The *_tks/*_ltks fields are whitespace-analyzed and store lowercase
	// tokens; a capitalized query term must not silently match nothing.
	query := buildQueryStringQuery(&types.MatchTextExpr{
		MatchingText: "Isabel Wood co-lead Ross Feldner Bob Musil Bird Watch Wonder Program",
	}, false, false)

	queryString, ok := query["query_string"].(map[string]interface{})
	if !ok {
		t.Fatalf("query_string missing from %#v", query)
	}
	assertEqual(t, queryString["query"],
		"isabel wood co-lead ross feldner bob musil bird watch wonder program")
}

func TestBuildQueryStringQueryKeepsBooleanOperators(t *testing.T) {
	// Only AND/OR/NOT are operators, and only in upper case: a folded "or" is a
	// term clause, which re-bases minimum_should_match and turns a 94-hit search
	// into a 32-hit one (see lowerCaseQueryText).
	query := buildQueryStringQuery(&types.MatchTextExpr{
		MatchingText: `(病毒 OR 勒索)^0.3 (系统)^0.2 ("系统 因为 勒索"~2)^1.5 OR ("MES"^0.7)`,
	}, false, false)

	queryString, ok := query["query_string"].(map[string]interface{})
	if !ok {
		t.Fatalf("query_string missing from %#v", query)
	}
	assertEqual(t, queryString["query"],
		`(病毒 OR 勒索)^0.3 (系统)^0.2 ("系统 因为 勒索"~2)^1.5 OR ("mes"^0.7)`)
}

func TestSearchUsesConfiguredKNNNumCandidates(t *testing.T) {
	if err := common.InitLogger("info", common.FileOutput{}, "elasticsearch_test"); err != nil {
		t.Fatalf("init logger: %v", err)
	}
	var searchQuery map[string]interface{}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if err := json.NewDecoder(r.Body).Decode(&searchQuery); err != nil {
			t.Errorf("decode search query: %v", err)
		}
		w.Header().Set("X-Elastic-Product", "Elasticsearch")
		w.Header().Set("Content-Type", "application/json")
		_, _ = w.Write([]byte(`{"hits":{"total":{"value":0},"hits":[]}}`))
	}))
	defer server.Close()

	client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
	if err != nil {
		t.Fatalf("new elasticsearch client: %v", err)
	}
	engine := &Engine{client: client}
	_, err = engine.Search(t.Context(), &types.SearchRequest{
		IndexNames: []string{"ragflow_tenant"},
		KbIDs:      []string{"kb-1"},
		Limit:      30,
		Filter: map[string]interface{}{
			"doc_id":        []string{"doc-1"},
			"available_int": 1,
			"category_kwd":  "allowed",
		},
		MatchExprs: []interface{}{&types.MatchDenseExpr{
			VectorColumnName: "q_2_vec",
			EmbeddingData:    []float64{0.1, 0.2},
			TopN:             128,
			ExtraOptions:     map[string]interface{}{"num_candidates": 4096},
		}},
	})
	if err != nil {
		t.Fatalf("Search: %v", err)
	}
	knn, ok := searchQuery["knn"].(map[string]interface{})
	if !ok {
		t.Fatalf("KNN query missing from %#v", searchQuery)
	}
	if knn["k"] != float64(128) || knn["num_candidates"] != float64(4096) {
		t.Fatalf("KNN parameters = (%v, %v), want (128, 4096)", knn["k"], knn["num_candidates"])
	}
	filterJSON, err := json.Marshal(knn["filter"])
	if err != nil {
		t.Fatal(err)
	}
	filter := string(filterJSON)
	for _, scope := range []string{"kb-1", "doc-1", "available_int", "category_kwd", "allowed"} {
		if !strings.Contains(filter, scope) {
			t.Fatalf("KNN filter %s lost scope %q", filter, scope)
		}
	}
	if strings.Contains(filter, "query_string") {
		t.Fatalf("dense-only KNN filter contains lexical predicate: %s", filter)
	}
}

func TestKNNScoresSearchesAllCandidateIndices(t *testing.T) {
	searches := 0
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		searches++
		if r.URL.Path != "/ragflow_a,ragflow_b/_search" {
			t.Errorf("KNN search path = %s, want both candidate indices once", r.URL.Path)
		}
		if r.URL.Query().Get("allow_partial_search_results") != "false" {
			t.Error("partial KNN results must trigger the local reranking fallback")
		}
		var query map[string]interface{}
		if err := json.NewDecoder(r.Body).Decode(&query); err != nil {
			t.Error(err)
		}
		knn, _ := query["knn"].(map[string]interface{})
		filter, _ := json.Marshal(knn["filter"])
		if string(filter) != `{"terms":{"id":["a1","a2","b1"]}}` {
			t.Errorf("KNN candidate filter = %s, want candidates from both indices", filter)
		}
		w.Header().Set("X-Elastic-Product", "Elasticsearch")
		_, _ = w.Write([]byte(`{"hits":{"hits":[{"_id":"a1","_score":0.9},{"_id":"a2","_score":0.8},{"_id":"b1","_score":0.7}]}}`))
	}))
	defer server.Close()
	eng := newTestEngine(t, server.URL)
	chunks := []map[string]interface{}{
		{"_id": "a1", "_index": "ragflow_a"},
		{"_id": "a2", "_index": "ragflow_a"},
		{"_id": "b1", "_index": "ragflow_b"},
	}
	result, err := eng.KNNScores(t.Context(), chunks, []float64{0.1, 0.2}, 3)
	if err != nil {
		t.Fatal(err)
	}
	if got := eng.GetScores(result); !reflect.DeepEqual(got, map[string]float64{"a1": 0.9, "a2": 0.8, "b1": 0.7}) {
		t.Fatalf("KNN scores = %v, want scores from both indices", got)
	}
	result, err = eng.KNNScores(t.Context(), []map[string]interface{}{{"_id": "unscoped"}}, []float64{0.1, 0.2}, 1)
	if err == nil || result != nil || searches != 1 {
		t.Fatalf("missing index: result = %v, err = %v, searches = %d, want fallback without a cluster-wide query", result, err, searches)
	}
}

func TestUpdateSingleMemoryMessageWaitsForRefresh(t *testing.T) {
	var gotRefresh string
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/memory_tenant/_update/memory-1_42" {
			t.Errorf("path=%s, want /memory_tenant/_update/memory-1_42", r.URL.Path)
			http.Error(w, "unexpected request path", http.StatusNotFound)
			return
		}
		gotRefresh = r.URL.Query().Get("refresh")
		w.Header().Set("X-Elastic-Product", "Elasticsearch")
		w.Header().Set("Content-Type", "application/json")
		_, _ = w.Write([]byte(`{"result":"updated"}`))
	}))
	defer server.Close()

	client, err := elasticsearch.NewClient(elasticsearch.Config{
		Addresses: []string{server.URL},
	})
	if err != nil {
		t.Fatalf("new elasticsearch client: %v", err)
	}
	ctx := t.Context()
	engine := &Engine{client: client}
	if err = engine.updateSingleMemoryMessage(ctx, "memory_tenant", "memory-1_42", map[string]interface{}{"forget_at": "2026-07-27 10:00:00"}); err != nil {
		t.Fatalf("updateSingleMemoryMessage: %v", err)
	}
	if gotRefresh != "wait_for" {
		t.Fatalf("refresh=%q, want wait_for", gotRefresh)
	}
}

func TestDeleteChunksPreservesStringSliceCondition(t *testing.T) {
	var deleteQuery map[string]interface{}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("X-Elastic-Product", "Elasticsearch")
		switch r.Method {
		case http.MethodHead:
			w.WriteHeader(http.StatusOK)
		case http.MethodPost:
			if r.URL.Path != "/ragflow_tenant/_delete_by_query" {
				t.Errorf("path=%s, want /ragflow_tenant/_delete_by_query", r.URL.Path)
				http.Error(w, "unexpected request path", http.StatusNotFound)
				return
			}
			if err := json.NewDecoder(r.Body).Decode(&deleteQuery); err != nil {
				t.Errorf("decode delete query: %v", err)
			}
			w.Header().Set("Content-Type", "application/json")
			_, _ = w.Write([]byte(`{"deleted":3}`))
		default:
			t.Errorf("method=%s", r.Method)
			http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
		}
	}))
	defer server.Close()
	ctx := t.Context()
	client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
	if err != nil {
		t.Fatalf("new elasticsearch client: %v", err)
	}
	engine := &Engine{client: client}
	deleted, err := engine.DeleteChunks(ctx, map[string]interface{}{
		"kb_id":          "kb-1",
		"source_doc_ids": []string{"doc-a", "doc-b"},
	}, "ragflow_tenant", "kb-1")
	if err != nil {
		t.Fatalf("DeleteChunks: %v", err)
	}
	if deleted != 3 {
		t.Fatalf("deleted=%d, want 3", deleted)
	}

	query, ok := deleteQuery["query"].(map[string]interface{})
	if !ok {
		t.Fatalf("delete query missing query: %#v", deleteQuery)
	}
	boolQuery, ok := query["bool"].(map[string]interface{})
	if !ok {
		t.Fatalf("delete query missing bool: %#v", query)
	}
	must, ok := boolQuery["must"].([]interface{})
	if !ok || len(must) != 1 {
		t.Fatalf("delete query must=%#v, want one terms clause", boolQuery["must"])
	}
	terms := must[0].(map[string]interface{})["terms"].(map[string]interface{})
	assertEqual(t, terms["source_doc_ids"], []interface{}{"doc-a", "doc-b"})
}

// TestDeleteChunksIDStringSlice guards against the id filter being dropped when
// the caller passes a typed []string. Without a dedicated []string branch in
// the id handling, the generic loop skips the "id" key and DeleteChunks would
// build a match_all query and delete every document in the index.
func TestDeleteChunksIDStringSlice(t *testing.T) {
	var deleteQuery map[string]interface{}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("X-Elastic-Product", "Elasticsearch")
		switch r.Method {
		case http.MethodHead:
			w.WriteHeader(http.StatusOK)
		case http.MethodPost:
			if r.URL.Path != "/ragflow_tenant/_delete_by_query" {
				t.Errorf("path=%s, want /ragflow_tenant/_delete_by_query", r.URL.Path)
				http.Error(w, "unexpected request path", http.StatusNotFound)
				return
			}
			if err := json.NewDecoder(r.Body).Decode(&deleteQuery); err != nil {
				t.Errorf("decode delete query: %v", err)
			}
			w.Header().Set("Content-Type", "application/json")
			_, _ = w.Write([]byte(`{"deleted":2}`))
		default:
			t.Errorf("method=%s", r.Method)
			http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
		}
	}))
	defer server.Close()
	ctx := t.Context()
	client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
	if err != nil {
		t.Fatalf("new elasticsearch client: %v", err)
	}
	engine := &Engine{client: client}
	if _, err = engine.DeleteChunks(ctx, map[string]interface{}{
		"id": []string{"doc-a", "doc-b"},
	}, "ragflow_tenant", "kb-1"); err != nil {
		t.Fatalf("DeleteChunks: %v", err)
	}

	query, ok := deleteQuery["query"].(map[string]interface{})
	if !ok {
		t.Fatalf("delete query missing query: %#v", deleteQuery)
	}
	boolQuery, ok := query["bool"].(map[string]interface{})
	if !ok {
		t.Fatalf("delete query missing bool: %#v", query)
	}
	must, ok := boolQuery["must"].([]interface{})
	if !ok || len(must) != 1 {
		t.Fatalf("delete query must=%#v, want one id terms clause (match_all would be a bug)", boolQuery["must"])
	}
	terms, ok := must[0].(map[string]interface{})["terms"].(map[string]interface{})
	if !ok {
		t.Fatalf("must[0] missing terms clause: %#v", must[0])
	}
	assertEqual(t, terms["id"], []interface{}{"doc-a", "doc-b"})
}

// TestDeleteChunksScopesByDataset pins the engine-side dataset scope. The ES
// index is shared by every dataset of a tenant, so DeleteChunks must turn the
// datasetID argument into a kb_id filter even when the caller's condition does
// not carry one; otherwise a doc_id-only delete reaches other datasets.
func TestDeleteChunksScopesByDataset(t *testing.T) {
	cases := []struct {
		name      string
		condition map[string]interface{}
	}{
		{name: "condition without kb_id", condition: map[string]interface{}{"doc_id": "doc-1"}},
		{name: "datasetID wins over conflicting kb_id", condition: map[string]interface{}{"doc_id": "doc-1", "kb_id": "kb-2"}},
		{name: "typed kb_id slice cannot widen the scope", condition: map[string]interface{}{"id": []string{"chunk-1"}, "kb_id": []string{"kb-2"}}},
		{name: "empty condition stays dataset-scoped", condition: nil},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			var deleteQuery map[string]interface{}
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("X-Elastic-Product", "Elasticsearch")
				switch r.Method {
				case http.MethodHead:
					w.WriteHeader(http.StatusOK)
				case http.MethodPost:
					if r.URL.Path != "/ragflow_tenant/_delete_by_query" {
						t.Errorf("path=%s, want /ragflow_tenant/_delete_by_query", r.URL.Path)
						http.Error(w, "unexpected request path", http.StatusNotFound)
						return
					}
					if err := json.NewDecoder(r.Body).Decode(&deleteQuery); err != nil {
						t.Errorf("decode delete query: %v", err)
					}
					w.Header().Set("Content-Type", "application/json")
					_, _ = w.Write([]byte(`{"deleted":1}`))
				default:
					t.Errorf("method=%s", r.Method)
					http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
				}
			}))
			defer server.Close()
			client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
			if err != nil {
				t.Fatalf("new elasticsearch client: %v", err)
			}
			engine := &Engine{client: client}
			if _, err = engine.DeleteChunks(t.Context(), tc.condition, "ragflow_tenant", "kb-1"); err != nil {
				t.Fatalf("DeleteChunks: %v", err)
			}

			query, _ := json.Marshal(deleteQuery)
			if string(query) == `{"query":{"match_all":{}}}` {
				t.Fatalf("empty condition must stay dataset-scoped, got %s", query)
			}
			if !strings.Contains(string(query), `"kb_id":"kb-1"`) {
				t.Fatalf("delete query lost dataset scope kb-1: %s", query)
			}
			if strings.Contains(string(query), `"kb_id":"kb-2"`) {
				t.Fatalf("delete query kept conflicting kb_id kb-2: %s", query)
			}
		})
	}
}

// TestUpdateChunksPreservesStringSliceCondition guards the document
// availability switch: the doc-service caller of UpdateChunks
// (updateDocumentChunkAvailability) passes a typed []string id list, and a
// builder that only understands []interface{} silently drops the id clause,
// widening the update-by-query to every chunk of the dataset (kb_id is then the
// only remaining filter).
func TestUpdateChunksPreservesStringSliceCondition(t *testing.T) {
	var updateQuery map[string]interface{}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("X-Elastic-Product", "Elasticsearch")
		switch r.Method {
		case http.MethodHead:
			w.WriteHeader(http.StatusOK)
		case http.MethodPost:
			if r.URL.Path != "/ragflow_tenant/_update_by_query" {
				t.Errorf("path=%s, want /ragflow_tenant/_update_by_query", r.URL.Path)
				http.Error(w, "unexpected request path", http.StatusNotFound)
				return
			}
			if err := json.NewDecoder(r.Body).Decode(&updateQuery); err != nil {
				t.Errorf("decode update query: %v", err)
			}
			w.Header().Set("Content-Type", "application/json")
			_, _ = w.Write([]byte(`{"updated":2}`))
		default:
			t.Errorf("method=%s", r.Method)
			http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
		}
	}))
	defer server.Close()

	ctx := t.Context()
	client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
	if err != nil {
		t.Fatalf("new elasticsearch client: %v", err)
	}
	engine := &Engine{client: client}
	if err = engine.UpdateChunks(ctx,
		map[string]interface{}{"id": []string{"chunk-a", "chunk-b"}},
		map[string]interface{}{"available_int": 1, "source_doc_ids": []string{"doc-a"}},
		"ragflow_tenant", "kb-1"); err != nil {
		t.Fatalf("UpdateChunks: %v", err)
	}

	query, ok := updateQuery["query"].(map[string]interface{})
	if !ok {
		t.Fatalf("update query missing query: %#v", updateQuery)
	}
	boolQuery, ok := query["bool"].(map[string]interface{})
	if !ok {
		t.Fatalf("update query missing bool: %#v", query)
	}
	filter, ok := boolQuery["filter"].([]interface{})
	if !ok {
		t.Fatalf("update query missing filter: %#v", boolQuery)
	}
	var gotIDs, gotKB interface{}
	for _, raw := range filter {
		clause, ok := raw.(map[string]interface{})
		if !ok {
			continue
		}
		if terms, ok := clause["terms"].(map[string]interface{}); ok {
			if ids, ok := terms["id"]; ok {
				gotIDs = ids
			}
		}
		if term, ok := clause["term"].(map[string]interface{}); ok {
			if kb, ok := term["kb_id"]; ok {
				gotKB = kb
			}
		}
	}
	if gotIDs == nil {
		t.Fatalf("update filter dropped the id terms clause (would hit the whole dataset): %#v", filter)
	}
	assertEqual(t, gotIDs, []interface{}{"chunk-a", "chunk-b"})
	assertEqual(t, gotKB, "kb-1")

	script, ok := updateQuery["script"].(map[string]interface{})
	if !ok {
		t.Fatalf("update query missing script: %#v", updateQuery)
	}
	source, _ := script["source"].(string)
	if !strings.Contains(source, "ctx._source.available_int=1;") {
		t.Fatalf("script source=%q, want available_int assignment", source)
	}
	if !strings.Contains(source, "ctx._source.source_doc_ids=params.pp_source_doc_ids;") {
		t.Fatalf("script source=%q, want typed []string value assignment", source)
	}
	scriptParams, ok := script["params"].(map[string]interface{})
	if !ok {
		t.Fatalf("update script missing params: %#v", script)
	}
	assertEqual(t, scriptParams["pp_source_doc_ids"], []interface{}{"doc-a"})
}

// TestUpdateChunksPreservesMustNotCondition locks the exclusion half of the
// condition contract. A dropped must_not widens the update to exactly the rows
// the caller asked to exclude (e.g. hybrid_search's
// must_not={"exists":"compile_kwd"} excluding compiled products).
func TestUpdateChunksPreservesMustNotCondition(t *testing.T) {
	var updateQuery map[string]interface{}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("X-Elastic-Product", "Elasticsearch")
		switch r.Method {
		case http.MethodHead:
			w.WriteHeader(http.StatusOK)
		case http.MethodPost:
			if r.URL.Path != "/ragflow_tenant/_update_by_query" {
				t.Errorf("path=%s, want /ragflow_tenant/_update_by_query", r.URL.Path)
				http.Error(w, "unexpected request path", http.StatusNotFound)
				return
			}
			if err := json.NewDecoder(r.Body).Decode(&updateQuery); err != nil {
				t.Errorf("decode update query: %v", err)
			}
			w.Header().Set("Content-Type", "application/json")
			_, _ = w.Write([]byte(`{"updated":1}`))
		default:
			t.Errorf("method=%s", r.Method)
			http.Error(w, "unexpected method", http.StatusMethodNotAllowed)
		}
	}))
	defer server.Close()

	ctx := t.Context()
	client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
	if err != nil {
		t.Fatalf("new elasticsearch client: %v", err)
	}
	engine := &Engine{client: client}
	exclusion := []interface{}{
		map[string]interface{}{"exists": map[string]interface{}{"field": "compile_kwd"}},
	}
	if err = engine.UpdateChunks(ctx,
		map[string]interface{}{
			"doc_id":   "doc-1",
			"must_not": map[string]interface{}{"exists": "compile_kwd"},
		},
		map[string]interface{}{"available_int": 0},
		"ragflow_tenant", "kb-1"); err != nil {
		t.Fatalf("UpdateChunks: %v", err)
	}
	boolQuery := mustBoolQuery(t, updateQuery)
	assertEqual(t, boolQuery["must_not"], exclusion)
	if _, ok := boolQuery["filter"]; !ok {
		t.Fatalf("update query lost the positive filter: %#v", boolQuery)
	}

	// A must_not-only condition is an explicit "everything except" scope rather
	// than a dropped filter, so it is accepted and forwarded. DeleteChunks
	// behaves the same way.
	if err = engine.updateChunksByQuery(ctx, "ragflow_tenant",
		map[string]interface{}{"must_not": map[string]interface{}{"exists": "compile_kwd"}},
		map[string]interface{}{"available_int": 0}); err != nil {
		t.Fatalf("updateChunksByQuery must_not only: %v", err)
	}
	boolQuery = mustBoolQuery(t, updateQuery)
	assertEqual(t, boolQuery["must_not"], exclusion)
	if _, ok := boolQuery["filter"]; ok {
		t.Fatalf("must_not-only update should carry no positive filter: %#v", boolQuery)
	}
}

// mustBoolQuery extracts query.bool from a captured update-by-query body.
func mustBoolQuery(t *testing.T, body map[string]interface{}) map[string]interface{} {
	t.Helper()
	query, ok := body["query"].(map[string]interface{})
	if !ok {
		t.Fatalf("update query missing query: %#v", body)
	}
	boolQuery, ok := query["bool"].(map[string]interface{})
	if !ok {
		t.Fatalf("update query missing bool: %#v", query)
	}
	return boolQuery
}

// TestUpdateChunksByQueryRejectsUnfilteredUpdate locks the guard: a non-empty
// condition whose keys all get dropped must fail instead of rewriting every row
// of the index.
func TestUpdateChunksByQueryRejectsUnfilteredUpdate(t *testing.T) {
	engine := &Engine{}
	err := engine.updateChunksByQuery(t.Context(), "ragflow_tenant",
		map[string]interface{}{"unsupported_field": struct{}{}},
		map[string]interface{}{"available_int": 1})
	if err == nil {
		t.Fatal("updateChunksByQuery accepted a condition that yields no filter clause")
	}
	if !strings.Contains(err.Error(), "unfiltered update") {
		t.Fatalf("err=%v, want unfiltered update refusal", err)
	}
}

func TestElasticsearchGetFieldsFiltersAndUsesIDFallback(t *testing.T) {
	engine := &Engine{}
	chunks := []map[string]interface{}{
		{
			"_id":                 "fallback-chunk",
			"docnm_kwd":           []interface{}{"guide.md"},
			"content_with_weight": "Alpha beta body.",
			"available_int":       float64(1),
			"ignored":             "not requested",
		},
	}

	got := engine.GetFields(chunks, []string{"id", "docnm_kwd", "content_with_weight", "available_int"})
	fieldMap, ok := got["fallback-chunk"]
	if !ok {
		t.Fatalf("GetFields keys=%v, want fallback-chunk", got)
	}

	assertEqual(t, fieldMap["id"], "fallback-chunk")
	assertEqual(t, fieldMap["docnm_kwd"], "guide.md")
	assertEqual(t, fieldMap["content_with_weight"], "Alpha beta body.")
	assertEqual(t, fieldMap["available_int"], float64(1))

	if _, ok := fieldMap["ignored"]; ok {
		t.Fatalf("field filter leaked unrequested field: %#v", fieldMap)
	}
}

// TestBuildRankFeatureQueryUsesTopLevelPagerank pins the field naming: tag
// weights address keys inside the tag_feas map, while pagerank addresses the
// top-level pagerank_fea rank_feature field.
func TestBuildRankFeatureQueryUsesTopLevelPagerank(t *testing.T) {
	queries := buildRankFeatureQuery(map[string]float64{
		common.PAGERANK_FLD: 10,
		"tag_a":             0.8,
	})
	if len(queries) != 2 {
		t.Fatalf("queries = %d, want 2", len(queries))
	}
	byField := make(map[string]map[string]interface{}, len(queries))
	for _, q := range queries {
		clause, ok := q["rank_feature"].(map[string]interface{})
		if !ok {
			t.Fatalf("not a rank_feature clause: %#v", q)
		}
		field, _ := clause["field"].(string)
		byField[field] = clause
	}
	pagerank, ok := byField[common.PAGERANK_FLD]
	if !ok {
		t.Fatalf("pagerank clause missing, got fields %#v", byField)
	}
	if got := pagerank["boost"]; got != float64(10) {
		t.Fatalf("pagerank boost = %v, want 10", got)
	}
	tagField := common.TAG_FLD + ".tag_a"
	if _, ok := byField[tagField]; !ok {
		t.Fatalf("tag clause %q missing, got fields %#v", tagField, byField)
	}
}

// TestElasticsearchGetFieldsEncodesNestedObjects covers the rank-feature
// fields: tag_feas is a "rank_features" object in _source, so the projection
// has to render it as JSON (the reranker parses JSON, not Go's "%v" map form),
// while numeric fields keep the stringified form the rest of the projection uses.
func TestElasticsearchGetFieldsEncodesNestedObjects(t *testing.T) {
	engine := &Engine{}
	chunks := []map[string]interface{}{
		{
			"id":                  "chunk-1",
			"tag_feas":            map[string]interface{}{"tag_a": float64(0.5)},
			"pagerank_fea":        float64(7),
			"content_with_weight": "body",
		},
	}

	fieldMap, ok := engine.GetFields(chunks, []string{"id", "tag_feas", "pagerank_fea"})["chunk-1"]
	if !ok {
		t.Fatalf("GetFields lost chunk-1")
	}
	assertEqual(t, fieldMap["tag_feas"], `{"tag_a":0.5}`)
	assertEqual(t, fieldMap["pagerank_fea"], "7")
}

func TestElasticsearchGetFieldsEmptyAndSkippedIDs(t *testing.T) {
	engine := &Engine{}

	if got := engine.GetFields(nil, nil); got == nil || len(got) != 0 {
		t.Fatalf("GetFields(nil)=%#v, want empty non-nil map", got)
	}

	got := engine.GetFields([]map[string]interface{}{
		{"id": "chunk-1", "docnm_kwd": "doc.md"},
		{"id": "", "_id": "fallback-chunk", "docnm_kwd": "fallback.md"},
		{"docnm": "missing-id.md"},
	}, []string{"id", "docnm_kwd"})

	fieldMap, ok := got["chunk-1"]
	if !ok {
		t.Fatalf("GetFields keys=%v, want chunk-1", got)
	}
	assertEqual(t, fieldMap["id"], "chunk-1")
	assertEqual(t, fieldMap["docnm_kwd"], "doc.md")

	if _, ok := got["missing-id.md"]; ok {
		t.Fatalf("chunk without id should be skipped: %#v", got)
	}
	fallbackMap, ok := got["fallback-chunk"]
	if !ok {
		t.Fatalf("GetFields keys=%v, want fallback-chunk", got)
	}

	assertEqual(t, fallbackMap["id"], "fallback-chunk")
	assertEqual(t, fallbackMap["docnm_kwd"], "fallback.md")
}

func TestElasticsearchGetAggregationSplitsCountsAndSorts(t *testing.T) {
	engine := &Engine{}
	chunks := []map[string]interface{}{
		{"tag_kwd": "red###blue###"},
		{"tag_kwd": []interface{}{"blue", " green ", ""}},
		{"tag_kwd": "blue"},
		{"tag_kwd": ""},
		{},
	}

	got := engine.GetAggregation(chunks, "tag_kwd")
	assertEqual(t, aggregationCounts(t, got), map[string]int{"blue": 3, "red": 1, "green": 1})
	if len(got) == 0 || got[0]["key"] != "blue" || got[0]["count"] != 3 {
		t.Fatalf("first aggregation=%#v, want blue count 3", got)
	}
	if len(got) != 3 || got[1]["key"] != "green" || got[2]["key"] != "red" {
		t.Fatalf("tie ordering=%#v, want green before red", got)
	}

	docAgg := engine.GetAggregation([]map[string]interface{}{
		{"docnm_kwd": "guide.md, api.md"},
		{"docnm_kwd": "guide.md"},
	}, "docnm_kwd")
	assertEqual(t, aggregationCounts(t, docAgg), map[string]int{"guide.md": 2, "api.md": 1})

	if got := engine.GetAggregation(chunks, "missing_kwd"); got == nil || len(got) != 0 {
		t.Fatalf("missing aggregation=%#v, want empty non-nil slice", got)
	}
}

func TestElasticsearchGetChunkIDsPreservesOrderWithFallback(t *testing.T) {
	engine := &Engine{}
	chunks := []map[string]interface{}{
		{"id": "source-id", "_id": "hit-id"},
		{"_id": "fallback-id"},
		{"id": ""},
		{"id": 42},
		{"id": "last-id"},
	}

	got := engine.GetChunkIDs(chunks)
	assertEqual(t, got, []string{"source-id", "fallback-id", "last-id"})

	if got := engine.GetChunkIDs(nil); got == nil || len(got) != 0 {
		t.Fatalf("GetChunkIDs(nil)=%#v, want empty non-nil slice", got)
	}
}

func TestElasticsearchGetHighlightUsesRegex(t *testing.T) {
	engine := &Engine{}
	chunks := []map[string]interface{}{
		{
			"_id":                 "full-text",
			"content_with_weight": "Alpha beta.\nbetamax soup. BETA again!",
		},
		{"id": "no-match", "content_with_weight": "Keep the complete text."},
		{"id": "missing"},
	}

	got := engine.GetHighlight(chunks, []string{"beta"}, "content_with_weight")
	assertEqual(t, got, map[string]string{
		"full-text": "Alpha <em>beta</em>.\n<em>betamax</em> soup. <em>BETA</em> again!",
		"no-match":  "Keep the complete text.",
	})
}

func TestElasticsearchGetHighlightExpandsLatinStems(t *testing.T) {
	engine := &Engine{}
	chunks := []map[string]interface{}{
		{
			"_id":                 "latin",
			"content_with_weight": "Required coordinated isolation. Coordinación.",
		},
	}

	got := engine.GetHighlight(chunks, []string{"requir", "coordin", "isol"}, "content_with_weight")
	assertEqual(t, got, map[string]string{
		"latin": "<em>Required</em> <em>coordinated</em> <em>isolation</em>. <em>Coordinación</em>.",
	})
}

func TestElasticsearchGetHighlightPrefersLongKeywords(t *testing.T) {
	engine := &Engine{}

	gotExisting := engine.GetHighlight([]map[string]interface{}{
		{"id": "existing", "content_with_weight": "already <em>marked</em> text"},
	}, []string{"marked"}, "content_with_weight")
	assertEqual(t, gotExisting, map[string]string{"existing": "already <em><em>marked</em></em> text"})

	gotNonEnglish := engine.GetHighlight([]map[string]interface{}{
		{"id": "cn", "content_with_weight": "这是世界。你好世界"},
	}, []string{"世界"}, "content_with_weight")
	assertEqual(t, gotNonEnglish, map[string]string{
		"cn": "这是<em>世界</em>。你好<em>世界</em>",
	})

	gotOverlapping := engine.GetHighlight([]map[string]interface{}{
		{"id": "overlap", "content_with_weight": "世界和世"},
	}, []string{"世", "世界", "世界"}, "content_with_weight")
	assertEqual(t, gotOverlapping, map[string]string{
		"overlap": "<em>世界</em>和<em>世</em>",
	})

	if got := engine.GetHighlight([]map[string]interface{}{{"id": "x"}}, []string{"x"}, "content_with_weight"); got == nil || len(got) != 0 {
		t.Fatalf("missing field highlight=%#v, want empty non-nil map", got)
	}
	if got := engine.GetHighlight([]map[string]interface{}{{"id": "x", "content": "x"}}, nil, "content"); !reflect.DeepEqual(got, map[string]string{"x": "x"}) {
		t.Fatalf("empty keyword highlight=%#v, want unchanged text", got)
	}
}

func aggregationCounts(t *testing.T, aggregation []map[string]interface{}) map[string]int {
	t.Helper()

	counts := make(map[string]int, len(aggregation))
	for _, item := range aggregation {
		key, ok := item["key"].(string)
		if !ok {
			t.Fatalf("aggregation key type=%T in %#v", item["key"], item)
		}
		count, ok := item["count"].(int)
		if !ok {
			t.Fatalf("aggregation count type=%T in %#v", item["count"], item)
		}
		counts[key] = count
	}
	return counts
}

func assertEqual(t *testing.T, got, want interface{}) {
	t.Helper()
	if !reflect.DeepEqual(got, want) {
		t.Fatalf("got %#v, want %#v", got, want)
	}
}

// TestMapMemoryMessageToESDoc: logical message fields are mapped to the
// storage document at insert time — message_type/status collapse to
// their storage counterparts and content is tokenized before write,
// mirroring Python memory/utils/es_conn.py:map_message_to_es_fields.
func TestMapMemoryMessageToESDoc(t *testing.T) {
	doc := mapMemoryMessageToESDoc(map[string]interface{}{
		"id":           "mem-1_42",
		"doc_id":       "mem-1",
		"message_id":   int64(42),
		"message_type": "raw",
		"source_id":    0,
		"memory_id":    "mem-1",
		"agent_id":     "a1",
		"content":      "User Input: hello world\nAgent Response: hi there",
		"status":       true,
		"forget_at":    nil,
	})

	assertEqual(t, doc["message_type_kwd"], "raw")
	if _, ok := doc["message_type"]; ok {
		t.Fatalf("message_type must collapse into message_type_kwd, got %#v", doc)
	}
	assertEqual(t, doc["status_int"], 1)
	if _, ok := doc["status"]; ok {
		t.Fatalf("status must collapse into status_int, got %#v", doc)
	}

	raw := "User Input: hello world\nAgent Response: hi there"
	assertEqual(t, doc["content_ltks"], raw)
	tokenized, ok := doc["tokenized_content_ltks"].(string)
	if !ok || tokenized == "" {
		t.Fatalf("tokenized_content_ltks = %#v, want non-empty tokenized string", doc["tokenized_content_ltks"])
	}
	if _, ok := doc["content"]; ok {
		t.Fatalf("content must not be stored verbatim, got %#v", doc)
	}

	// Untouched fields pass through.
	assertEqual(t, doc["id"], "mem-1_42")
	assertEqual(t, doc["doc_id"], "mem-1")
	assertEqual(t, doc["message_id"], int64(42))
	assertEqual(t, doc["agent_id"], "a1")
	if v, ok := doc["forget_at"]; !ok || v != nil {
		t.Fatalf("forget_at = %#v, want explicit nil", v)
	}
}

// TestSearchKeepsRankFeatureOutOfKNNFilter pins the body-level split: the
// rank_feature clauses belong to the scored query, while knn.filter is a snapshot
// of the bool taken when the kNN clause is built and must keep only the scope and
// lexical predicates.
func TestSearchKeepsRankFeatureOutOfKNNFilter(t *testing.T) {
	if err := common.InitLogger("info", common.FileOutput{}, "elasticsearch_test"); err != nil {
		t.Fatalf("init logger: %v", err)
	}
	var searchQuery map[string]interface{}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if err := json.NewDecoder(r.Body).Decode(&searchQuery); err != nil {
			t.Errorf("decode search query: %v", err)
		}
		w.Header().Set("X-Elastic-Product", "Elasticsearch")
		w.Header().Set("Content-Type", "application/json")
		_, _ = w.Write([]byte(`{"hits":{"total":{"value":0},"hits":[]}}`))
	}))
	defer server.Close()

	client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
	if err != nil {
		t.Fatalf("new elasticsearch client: %v", err)
	}
	engine := &Engine{client: client}
	_, err = engine.Search(t.Context(), &types.SearchRequest{
		IndexNames: []string{"ragflow_tenant"},
		KbIDs:      []string{"kb-1"},
		Limit:      10,
		Filter:     map[string]interface{}{"available_int": 1},
		MatchExprs: []interface{}{
			&types.MatchTextExpr{MatchingText: "alpha", Fields: []string{"content_ltks"}, TopN: 10},
			&types.MatchDenseExpr{VectorColumnName: "q_2_vec", EmbeddingData: []float64{0.1, 0.2}, TopN: 128},
			&types.FusionExpr{Method: "weighted_sum", TopN: 10, FusionParams: map[string]interface{}{"weights": "0.7,0.3"}},
		},
		RankFeature: map[string]float64{"pagerank_fea": 10},
	})
	if err != nil {
		t.Fatalf("Search: %v", err)
	}

	queryJSON, err := json.Marshal(searchQuery["query"])
	if err != nil {
		t.Fatal(err)
	}
	if !strings.Contains(string(queryJSON), "rank_feature") || !strings.Contains(string(queryJSON), "pagerank_fea") {
		t.Fatalf("scored query lost its rank_feature clause: %s", queryJSON)
	}

	knn, ok := searchQuery["knn"].(map[string]interface{})
	if !ok {
		t.Fatalf("KNN query missing from %#v", searchQuery)
	}
	filterJSON, err := json.Marshal(knn["filter"])
	if err != nil {
		t.Fatal(err)
	}
	if strings.Contains(string(filterJSON), "rank_feature") {
		t.Fatalf("knn.filter carries rank_feature clauses: %s", filterJSON)
	}
	// The snapshot must still be the scope + lexical predicate, not an empty one.
	if !strings.Contains(string(filterJSON), "query_string") {
		t.Fatalf("knn.filter lost the lexical predicate: %s", filterJSON)
	}
}

// TestSearchKeepsEveryRankFeatureClause covers the multi-feature case: appending
// to the pre-loop should header used to keep only the last clause.
func TestSearchKeepsEveryRankFeatureClause(t *testing.T) {
	if err := common.InitLogger("info", common.FileOutput{}, "elasticsearch_test"); err != nil {
		t.Fatalf("init logger: %v", err)
	}
	var searchQuery map[string]interface{}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if err := json.NewDecoder(r.Body).Decode(&searchQuery); err != nil {
			t.Errorf("decode search query: %v", err)
		}
		w.Header().Set("X-Elastic-Product", "Elasticsearch")
		w.Header().Set("Content-Type", "application/json")
		_, _ = w.Write([]byte(`{"hits":{"total":{"value":0},"hits":[]}}`))
	}))
	defer server.Close()

	client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
	if err != nil {
		t.Fatalf("new elasticsearch client: %v", err)
	}
	engine := &Engine{client: client}
	_, err = engine.Search(t.Context(), &types.SearchRequest{
		IndexNames: []string{"ragflow_tenant"},
		KbIDs:      []string{"kb-1"},
		Limit:      10,
		Filter:     map[string]interface{}{"available_int": 1},
		MatchExprs: []interface{}{
			&types.MatchTextExpr{MatchingText: "alpha", Fields: []string{"content_ltks"}, TopN: 10},
			&types.MatchDenseExpr{VectorColumnName: "q_2_vec", EmbeddingData: []float64{0.1, 0.2}, TopN: 128},
			&types.FusionExpr{Method: "weighted_sum", TopN: 10, FusionParams: map[string]interface{}{"weights": "0.7,0.3"}},
		},
		RankFeature: map[string]float64{"tag_1": 0.8, "tag_2": 0.5},
	})
	if err != nil {
		t.Fatalf("Search: %v", err)
	}

	query, ok := searchQuery["query"].(map[string]interface{})
	if !ok {
		t.Fatalf("query missing from %#v", searchQuery)
	}
	boolMap, ok := query["bool"].(map[string]interface{})
	if !ok {
		t.Fatalf("bool query missing from %#v", query)
	}
	should, _ := boolMap["should"].([]interface{})
	var fields []string
	for _, clause := range should {
		clauseMap, ok := clause.(map[string]interface{})
		if !ok {
			continue
		}
		rankFeature, ok := clauseMap["rank_feature"].(map[string]interface{})
		if !ok {
			continue
		}
		if field, ok := rankFeature["field"].(string); ok {
			fields = append(fields, field)
		}
	}
	if len(fields) != 2 {
		t.Fatalf("rank_feature clauses = %v, want both tag fields", fields)
	}
	joined := strings.Join(fields, ",")
	if !strings.Contains(joined, "tag_feas.tag_1") || !strings.Contains(joined, "tag_feas.tag_2") {
		t.Fatalf("rank_feature fields = %v, want tag_feas.tag_1 and tag_feas.tag_2", fields)
	}
}

// TestSearchDoesNotAddPageRankToHitScore pins the hybrid score contract: _score is
// the ES hit score and nothing else. pagerank already reaches the hit score
// through the rank_feature clause and the final similarity through the reranker;
// folding it in here again would count it twice.
func TestSearchDoesNotAddPageRankToHitScore(t *testing.T) {
	if err := common.InitLogger("info", common.FileOutput{}, "elasticsearch_test"); err != nil {
		t.Fatalf("init logger: %v", err)
	}
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("X-Elastic-Product", "Elasticsearch")
		w.Header().Set("Content-Type", "application/json")
		_, _ = w.Write([]byte(`{"hits":{"total":{"value":1},"hits":[{"_id":"chunk-1","_index":"ragflow_tenant","_score":0.87,"_source":{"content_with_weight":"alpha","pagerank_fea":69,"tag_feas":{"tag_1":0.5}}}]}}`))
	}))
	defer server.Close()

	client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
	if err != nil {
		t.Fatalf("new elasticsearch client: %v", err)
	}
	engine := &Engine{client: client}
	res, err := engine.Search(t.Context(), &types.SearchRequest{
		IndexNames: []string{"ragflow_tenant"},
		KbIDs:      []string{"kb-1"},
		Limit:      10,
		Filter:     map[string]interface{}{"available_int": 1},
		MatchExprs: []interface{}{
			&types.MatchTextExpr{MatchingText: "alpha", Fields: []string{"content_ltks"}, TopN: 10},
			&types.MatchDenseExpr{VectorColumnName: "q_2_vec", EmbeddingData: []float64{0.1, 0.2}, TopN: 128},
			&types.FusionExpr{Method: "weighted_sum", TopN: 10, FusionParams: map[string]interface{}{"weights": "0.7,0.3"}},
		},
		RankFeature: map[string]float64{"pagerank_fea": 10},
	})
	if err != nil {
		t.Fatalf("Search: %v", err)
	}
	if len(res.Chunks) != 1 {
		t.Fatalf("Search returned %d chunks, want 1", len(res.Chunks))
	}
	if got, ok := res.Chunks[0]["_score"].(float64); !ok || got != 0.87 {
		t.Fatalf("_score = %#v, want the hit score 0.87 (pagerank must not be folded in)", res.Chunks[0]["_score"])
	}
	if got, ok := res.Chunks[0]["pagerank_fea"].(float64); !ok || got != 69 {
		t.Fatalf("pagerank_fea = %#v, want 69 preserved for the reranker", res.Chunks[0]["pagerank_fea"])
	}
}

// TestMapMemoryMessageESUpdateFieldsRefreshesTokens: writing content via
// the update path recomputes tokenized_content_ltks before write, same
// as the Python update path.
func TestMapMemoryMessageESUpdateFieldsRefreshesTokens(t *testing.T) {
	doc := mapMemoryMessageESUpdateFields(map[string]interface{}{
		"content": "fresh content",
		"status":  0,
	})

	assertEqual(t, doc["content_ltks"], "fresh content")
	tokenized, ok := doc["tokenized_content_ltks"].(string)
	if !ok || tokenized == "" {
		t.Fatalf("tokenized_content_ltks = %#v, want refreshed tokenized string", doc["tokenized_content_ltks"])
	}
	assertEqual(t, doc["status_int"], 0)
}

// TestDeleteChunksSurfacesPartialFailures guards against delete_by_query
// answering 200 with timed_out or failures set while DeleteChunks reports nil.
func TestDeleteChunksSurfacesPartialFailures(t *testing.T) {
	cases := map[string]string{
		"timed_out": `{"deleted":2,"timed_out":true,"failures":[]}`,
		"failures":  `{"deleted":2,"timed_out":false,"failures":[{"index":"ragflow_tenant","status":500}]}`,
	}
	for name, response := range cases {
		t.Run(name, func(t *testing.T) {
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("X-Elastic-Product", "Elasticsearch")
				if r.Method == http.MethodHead {
					w.WriteHeader(http.StatusOK)
					return
				}
				w.Header().Set("Content-Type", "application/json")
				_, _ = w.Write([]byte(response))
			}))
			defer server.Close()
			client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
			if err != nil {
				t.Fatalf("new elasticsearch client: %v", err)
			}
			engine := &Engine{client: client}
			deleted, err := engine.DeleteChunks(t.Context(), map[string]interface{}{"kb_id": "kb-1", "doc_id": "doc-1"}, "ragflow_tenant", "kb-1")
			if err == nil {
				t.Fatal("DeleteChunks returned nil error for a partial delete_by_query response")
			}
			if deleted != 2 {
				t.Fatalf("deleted=%d, want 2", deleted)
			}
		})
	}
}

// TestDeleteMetadataSurfacesPartialFailures guards the metadata side of the
// same contract: delete_by_query answering 200 with timed_out or failures set
// must surface as an error, otherwise RemoveDocumentKeepFile can leave orphan
// ragflow_doc_meta_<tenant> rows (#20659).
func TestDeleteMetadataSurfacesPartialFailures(t *testing.T) {
	cases := map[string]string{
		"timed_out": `{"deleted":2,"timed_out":true,"failures":[]}`,
		"failures":  `{"deleted":2,"timed_out":false,"failures":[{"index":"ragflow_doc_meta_tenant","status":500}]}`,
	}
	for name, response := range cases {
		t.Run(name, func(t *testing.T) {
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("X-Elastic-Product", "Elasticsearch")
				if r.Method == http.MethodHead {
					w.WriteHeader(http.StatusOK)
					return
				}
				w.Header().Set("Content-Type", "application/json")
				_, _ = w.Write([]byte(response))
			}))
			defer server.Close()
			client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
			if err != nil {
				t.Fatalf("new elasticsearch client: %v", err)
			}
			engine := &Engine{client: client}
			deleted, err := engine.DeleteMetadata(t.Context(), map[string]interface{}{"kb_id": "kb-1", "id": "doc-1"}, "tenant")
			if err == nil {
				t.Fatal("DeleteMetadata returned nil error for a partial delete_by_query response")
			}
			if deleted != 2 {
				t.Fatalf("deleted=%d, want 2", deleted)
			}
		})
	}
}

func TestCloneFilterSnapshotIsolatesLaterShouldAppends(t *testing.T) {
	boolMap := map[string]interface{}{
		"must":   []interface{}{map[string]interface{}{"term": map[string]interface{}{"kb_id": "kb-1"}}},
		"should": []interface{}{map[string]interface{}{"match_all": map[string]interface{}{}}},
	}
	boolQuery := map[string]interface{}{"bool": boolMap}

	snapshot := cloneFilterSnapshot(boolQuery)

	// Mirror what the rank_feature block does after the kNN clause is built: it
	// appends to the scored query's bool/should.
	boolMap["should"] = append(
		boolMap["should"].([]interface{}),
		map[string]interface{}{"rank_feature": map[string]interface{}{"field": "pagerank_fea", "boost": 10.0}},
	)

	if got := len(boolMap["should"].([]interface{})); got != 2 {
		t.Fatalf("original should has %d clauses, want 2", got)
	}
	snapshotBool, ok := snapshot["bool"].(map[string]interface{})
	if !ok {
		t.Fatalf("snapshot bool missing from %#v", snapshot)
	}
	if got := len(snapshotBool["should"].([]interface{})); got != 1 {
		t.Fatalf("snapshot should has %d clauses, want 1 (rank_feature must stay out of knn.filter)", got)
	}
	if got := len(snapshotBool["must"].([]interface{})); got != 1 {
		t.Fatalf("snapshot must has %d clauses, want 1", got)
	}
}

func TestCloneFilterSnapshotNil(t *testing.T) {
	if got := cloneFilterSnapshot(nil); got != nil {
		t.Fatalf("cloneFilterSnapshot(nil) = %#v, want nil", got)
	}
}

func TestCloneFilterSnapshotNonBoolQuery(t *testing.T) {
	query := map[string]interface{}{
		"query_string": map[string]interface{}{"query": "alpha"},
	}
	snapshot := cloneFilterSnapshot(query)
	if snapshot["query_string"] == nil {
		t.Fatalf("snapshot = %#v, want the query_string clause preserved", snapshot)
	}
}

func TestDeleteChunksRejectsEmptyDatasetID(t *testing.T) {
	for _, condition := range []map[string]interface{}{
		{"kb_id": "kb-2", "doc_id": "doc-1"},
		{"doc_id": "doc-1"},
		nil,
	} {
		var requests atomic.Int32
		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			requests.Add(1)
			w.Header().Set("X-Elastic-Product", "Elasticsearch")
			w.Header().Set("Content-Type", "application/json")
			if r.Method != http.MethodHead {
				_, _ = w.Write([]byte(`{"deleted":1}`))
			}
		}))
		client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
		if err != nil {
			server.Close()
			t.Fatal(err)
		}
		engine := &Engine{client: client}
		deleted, err := engine.DeleteChunks(t.Context(), condition, "ragflow_tenant", "")
		server.Close()
		if err == nil {
			t.Errorf("condition=%v: expected empty datasetID error, got deleted=%d", condition, deleted)
		}
		if deleted != 0 {
			t.Errorf("condition=%v: deleted=%d, want 0", condition, deleted)
		}
		if requests.Load() != 0 {
			t.Errorf("condition=%v: sent %d ES requests, want 0", condition, requests.Load())
		}
	}
}

func TestDeleteChunksRejectsEmptyIDs(t *testing.T) {
	cases := []struct {
		name string
		id   interface{}
	}{
		{name: "nil", id: nil},
		{name: "empty string slice", id: []string{}},
		{name: "nil string slice", id: []string(nil)},
		{name: "empty interface slice", id: []interface{}{}},
		{name: "nil interface slice", id: []interface{}(nil)},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			var deletes atomic.Int32
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				w.Header().Set("X-Elastic-Product", "Elasticsearch")
				w.Header().Set("Content-Type", "application/json")
				if r.Method == http.MethodPost {
					deletes.Add(1)
					_, _ = w.Write([]byte(`{"deleted":1}`))
				}
			}))
			defer server.Close()
			client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
			if err != nil {
				t.Fatal(err)
			}
			engine := &Engine{client: client}
			condition := map[string]interface{}{"id": tc.id}
			deleted, err := engine.DeleteChunks(t.Context(), condition, "ragflow_tenant", "kb-1")
			if err == nil {
				t.Error("expected an error for empty IDs")
			}
			if deleted != 0 {
				t.Errorf("deleted=%d, want 0", deleted)
			}
			if deletes.Load() != 0 {
				t.Errorf("sent %d delete requests, want 0", deletes.Load())
			}
		})
	}
}