Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

78.2 KB
Raw Blame
//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

package service

import (
	"context"
	"encoding/json"
	"fmt"
	"reflect"
	"sort"
	"strings"
	"testing"
	"time"

	"github.com/glebarez/sqlite"
	"gorm.io/gorm"

	"ragflow/internal/common"
	"ragflow/internal/dao"
	"ragflow/internal/engine"
	"ragflow/internal/entity"
	modelModule "ragflow/internal/entity/models"
	"ragflow/internal/permission"
)

// dialForTest builds a minimal *entity.Chat suitable for the
// guard-clause tests. KBs are empty so AsyncChat goes through
// AsyncChatSolo.
func dialForTest(llmid string) *entity.Chat {
	return &entity.Chat{
		ID:       "chat-1",
		TenantID: "tenant-1",
		LLMID:    llmid,
		PromptConfig: map[string]interface{}{
			"system":           "you are a test assistant.",
			"quote":            true,
			"refine_multiturn": false,
			"keyword":          false,
			"use_kg":           false,
			"toc_enhance":      false,
		},
		KBIDs:                  []interface{}{},
		VectorSimilarityWeight: 0.3,
	}
}

// newTimerAndPrompt builds a fresh Timer with all 6 phases recorded
// (with ~0 durations), so decorateAnswer emits the full Markdown
// block.
func newTimerAndPrompt() (*common.Timer, string) {
	t := common.NewTimer()
	t.Start()
	for _, p := range []common.Phase{
		common.PhaseCheckLLM,
		common.PhaseBindModels,
		common.PhaseRetrieval,
		common.PhaseGenerateAnswer,
	} {
		t.Enter(p)
		t.Exit(p)
	}
	return t, "Test prompt"
}

// --- P9 / P5 guard-clause tests on AsyncChat (P0 indirectly: input
//     validation runs before any RAG pipeline) ---

// TestAsyncChat_RejectsNonUserLastMessage covers the assertion at
// chat_pipeline.go:167. The OpenAI handler is supposed to enforce
// this, but a defense-in-depth check inside AsyncChat guards
// against misbehaving callers.
func TestAsyncChat_RejectsNonUserLastMessage(t *testing.T) {
	s := &ChatPipelineService{}
	messages := []map[string]interface{}{
		{"role": "user", "content": "first"},
		{"role": "assistant", "content": "last message must not be assistant"},
	}
	_, err := s.AsyncChat(t.Context(), dialForTest(""), messages, false, nil)
	if err == nil {
		t.Fatal("expected error for non-user last message, got nil")
	}
	if !strings.Contains(err.Error(), "not from user") {
		t.Errorf("unexpected error: %v", err)
	}
}

// TestAsyncChat_EmptyMessages covers the empty-messages case. The
// service should return an error before spawning the goroutine.
func TestAsyncChat_EmptyMessages(t *testing.T) {
	s := &ChatPipelineService{}
	_, err := s.AsyncChat(t.Context(), dialForTest(""), nil, false, nil)
	if err == nil {
		t.Fatal("expected error for empty messages, got nil")
	}
}

func TestAsyncChatRejectsNonStringUserContent(t *testing.T) {
	s := &ChatPipelineService{}
	messages := []map[string]interface{}{{"role": "user", "content": map[string]interface{}{"text": "question"}}}
	_, err := s.AsyncChat(t.Context(), dialForTest(""), messages, false, nil)
	if err == nil || !strings.Contains(err.Error(), "messages[0].content is invalid") {
		t.Fatalf("AsyncChat error = %v, want indexed invalid user content error", err)
	}
}

func TestValidateUserMessageContentPreservesTextParts(t *testing.T) {
	messages := []map[string]interface{}{{
		"role": "user",
		"content": []interface{}{
			map[string]interface{}{"type": "text", "text": "question"},
		},
	}}
	if err := validateUserMessageContent(messages); err != nil {
		t.Fatalf("valid text content parts rejected: %v", err)
	}
}

func TestValidateUserMessageContentRejectsNil(t *testing.T) {
	messages := []map[string]interface{}{{"role": "user", "content": nil}}
	if err := validateUserMessageContent(messages); err == nil || !strings.Contains(err.Error(), "messages[0].content") {
		t.Fatalf("validation error = %v, want indexed missing-content error", err)
	}
}

func TestAsyncChatRejectsUnauthorizedDatasetsBeforeStartingPipeline(t *testing.T) {
	db := setupChatRESTUpdateServiceTestDB(t)
	insertForeignPrivateDatasetForChatTest(t, db)

	chat := dialForTest("")
	chat.KBIDs = []interface{}{"kb-private"}
	results, err := (&ChatPipelineService{}).AsyncChat(
		serviceTestContext(t, "user-1", "user-1"),
		chat,
		[]map[string]interface{}{{"role": "user", "content": "show me the private dataset"}},
		false,
		nil,
	)
	if err == nil {
		for range results {
		}
		t.Fatal("AsyncChat accepted another user's private dataset")
	}
	if !isAccessDeniedOrHidden(err) {
		t.Fatalf("AsyncChat error = %v, want access denied or hidden resource", err)
	}
}

// TestBuildChatConfig_GenerationConfigReachesModel pins that the dialog's LLM
// setting and per-request overrides (temperature, top_p, max_tokens, thinking,
// stop) reach the model driver, and that request-level overrides win.
func TestBuildChatConfig_GenerationConfigReachesModel(t *testing.T) {
	chat := dialForTest("llm-1")
	chat.LLMSetting = entity.JSONMap{
		"temperature": 0.7,
		"top_p":       0.9,
		"max_tokens":  512,
		"thinking":    true,
		"stop":        []interface{}{"\n", "END"},
	}
	// Request-level overrides win over dialog values.
	cfg := BuildChatConfig(chat, map[string]interface{}{"temperature": 0.3})

	if cfg.Temperature == nil || *cfg.Temperature != 0.3 {
		t.Fatalf("Temperature: want request override 0.3, got %v", cfg.Temperature)
	}
	if cfg.TopP == nil || *cfg.TopP != 0.9 {
		t.Fatalf("TopP: want dialog 0.9, got %v", cfg.TopP)
	}
	if cfg.MaxTokens == nil || *cfg.MaxTokens != 512 {
		t.Fatalf("MaxTokens: want 512, got %v", cfg.MaxTokens)
	}
	if cfg.Thinking == nil || !*cfg.Thinking {
		t.Fatalf("Thinking: want true, got %v", cfg.Thinking)
	}
	if cfg.Stop == nil || len(*cfg.Stop) != 2 {
		t.Fatalf("Stop: want [\"\\n\", \"END\"], got %v", cfg.Stop)
	}
}

// --- P1 Timer + decorateAnswer tests (P0/P1/P7 surface) ---

// TestDecorateAnswer_TimerFormatAlwaysEmitted pins the Markdown
// layout of Timer, ensuring all six phase lines plus Total appear.
func TestDecorateAnswer_TimerFormatAlwaysEmitted(t *testing.T) {
	s := &ChatPipelineService{}
	timer, _ := newTimerAndPrompt()
	result := s.decorateAnswer(
		t.Context(),
		"hello world",
		map[string]interface{}{"chunks": []interface{}{}, "doc_aggs": []interface{}{}},
		"system prompt",
		[]string{"question"},
		0,
		timer,
		nil, 0.0, false,
		nil,
		"",
		nil,
		"",
		nil,
		false,
	)
	md := result.Prompt
	for _, must := range []string{
		"## Time elapsed:",
		"  - Check LLM:",
		"  - Bind models:",
		"  - Retrieval:",
		"  - Generate answer:",
		"  - Total:",
		"Generated tokens(approximately):",
	} {
		if !strings.Contains(md, must) {
			t.Errorf("decorateAnswer prompt missing %q in:\n%s", must, md)
		}
	}
}

// TestDecorateAnswer_ThinkMarkersPreserved covers the <think> split
// at decorateAnswer: when the LLM emits a think block, decorateAnswer
// moves the think block to the front of the final answer.
func TestDecorateAnswer_ThinkMarkersPreserved(t *testing.T) {
	s := &ChatPipelineService{}
	timer, _ := newTimerAndPrompt()
	result := s.decorateAnswer(
		t.Context(),
		"<think>reasoning</think>visible answer",
		map[string]interface{}{"chunks": []interface{}{}, "doc_aggs": []interface{}{}},
		"system prompt",
		[]string{"q"},
		0,
		timer,
		nil, 0.0, false,
		nil,
		"",
		nil,
		"",
		nil,
		false,
	)
	if !strings.HasPrefix(result.Answer, "<think>reasoning</think>") {
		t.Errorf("expected think block at start, got %q", result.Answer)
	}
	if !strings.Contains(result.Answer, "visible answer") {
		t.Errorf("expected visible answer in result, got %q", result.Answer)
	}
}

// TestDecorateAnswer_InvalidKeySuffix ensures the "Invalid API key"
// append path runs. This is an LLM error-marker check; the message
// survives cleanup.
func TestDecorateAnswer_InvalidKeySuffix(t *testing.T) {
	s := &ChatPipelineService{}
	timer, _ := newTimerAndPrompt()
	result := s.decorateAnswer(
		t.Context(),
		"oops: invalid api key",
		map[string]interface{}{"chunks": []interface{}{}, "doc_aggs": []interface{}{}},
		"system prompt",
		[]string{"q"},
		0,
		timer,
		nil, 0.0, false,
		nil,
		"",
		nil,
		"",
		nil,
		false,
	)
	if !strings.Contains(result.Answer, "Please set LLM API-Key") {
		t.Errorf("expected API-key hint, got %q", result.Answer)
	}
}

// TestDecorateAnswer_LeavesCanonicalMarkers covers P0: the decorator
// passes canonical [ID:N] markers through unchanged when quoting is enabled
// and there are no chunks to cite (so insertCitations is skipped).
func TestDecorateAnswer_LeavesCanonicalMarkers(t *testing.T) {
	s := &ChatPipelineService{}
	timer, _ := newTimerAndPrompt()
	result := s.decorateAnswer(
		t.Context(),
		"see [ID:12] for details",
		map[string]interface{}{"chunks": []interface{}{}, "doc_aggs": []interface{}{}},
		"system prompt",
		[]string{"q"},
		0,
		timer,
		nil, 0.0, true,
		nil,
		"",
		nil,
		"",
		nil,
		false,
	)
	if !strings.Contains(result.Answer, "[ID:12]") {
		t.Errorf("canonical marker must survive decorateAnswer, got %q", result.Answer)
	}
}

// TestDecorateAnswer_RepairNotRunWhenNoQuote covers P0.10: when
// quote=false, citation markers are removed without rewriting ordinary
// bracketed numbers.
func TestDecorateAnswer_RepairNotRunWhenNoQuote(t *testing.T) {
	s := &ChatPipelineService{}
	timer, _ := newTimerAndPrompt()
	result := s.decorateAnswer(
		t.Context(),
		"<think>Checking[ID:0]##0$$</think>see (ID: 12)[ID:0]##1$$ in [2024] for details",
		map[string]interface{}{"chunks": []interface{}{}, "doc_aggs": []interface{}{}},
		"system prompt",
		[]string{"q"},
		0,
		timer,
		nil, 0.0, false,
		nil,
		"",
		nil,
		"",
		nil,
		false,
	)
	if result.Answer != "<think>Checking</think>see  in [2024] for details" {
		t.Errorf("quote=false citation cleanup = %q", result.Answer)
	}
}

// TestDecorateAnswer_RepairRunsWhenQuote covers P0.10: when quote=true
// and the answer has bad citation shapes, RepairBadCitationFormats
// runs and produces canonical [ID:N] form.
func TestDecorateAnswer_RepairRunsWhenQuote(t *testing.T) {
	s := &ChatPipelineService{}
	timer, _ := newTimerAndPrompt()
	// Repair requires at least one chunk (mirrors Python's
	// `if knowledges and ...` guard). We provide one stub chunk so
	// the repair block runs.
	kb := map[string]interface{}{
		"chunks": []map[string]interface{}{
			map[string]interface{}{
				"chunk_id":            "c1",
				"content_with_weight": "hello world",
				"doc_id":              "d1",
			},
		},
		"doc_aggs": []interface{}{},
	}
	result := s.decorateAnswer(
		t.Context(),
		"see (ID: 12) for details",
		kb,
		"system prompt",
		[]string{"q"},
		0,
		timer,
		nil, 0.0, true, // quote=true
		nil,
		"",
		nil,
		"",
		nil,
		true,
	)
	if !strings.Contains(result.Answer, "[ID:12]") {
		t.Errorf("quote=true must repair to [ID:12], got %q", result.Answer)
	}
}

// TestDecorateAnswer_CitesThinkBlock ensures citations emitted inside the
// hidden thinking block still determine the returned reference documents.
func TestDecorateAnswer_CitesThinkBlock(t *testing.T) {
	s := &ChatPipelineService{}
	timer, _ := newTimerAndPrompt()
	kbinfos := map[string]interface{}{
		"chunks": []map[string]interface{}{
			{
				"chunk_id":            "c1",
				"content_with_weight": "hello world",
				"doc_id":              "d1",
			},
		},
		"doc_aggs": []interface{}{
			map[string]interface{}{"doc_id": "d1", "doc_name": "doc.txt"},
		},
	}
	result := s.decorateAnswer(
		t.Context(),
		"<think>Supported by [ID:0].</think>visible answer",
		kbinfos,
		"system prompt",
		[]string{"q"},
		0,
		timer,
		nil, 0.0, true,
		nil,
		"",
		nil,
		"",
		nil,
		true,
	)
	if result.Reference == nil {
		t.Fatal("a think-block citation must carry a reference")
	}
	if aggs, _ := result.Reference["doc_aggs"].([]interface{}); len(aggs) != 1 || aggs[0].(map[string]interface{})["doc_id"] != "d1" {
		t.Fatalf("reference doc_aggs = %#v, want the cited document", result.Reference["doc_aggs"])
	}
}

// TestDecorateAnswer_PreCheckSkipsInsertCitations covers P0.11: when
// the LLM already emitted canonical [ID:N] markers, insertCitations
// is skipped (so we don't double-tag). We verify by checking that
// the final answer keeps the same marker count we sent in.
func TestDecorateAnswer_PreCheckSkipsInsertCitations(t *testing.T) {
	s := &ChatPipelineService{}
	timer, _ := newTimerAndPrompt()
	in := "answer has [ID:3] already in it"
	result := s.decorateAnswer(
		t.Context(),
		in,
		map[string]interface{}{
			// No chunks → insertCitations path is gated off anyway,
			// but the pre-check still works on the answer.
			"chunks":   []map[string]interface{}{},
			"doc_aggs": []interface{}{},
		},
		"system prompt",
		[]string{"q"},
		0,
		timer,
		nil, 0.0, true,
		nil,
		"",
		nil,
		"",
		nil,
		false,
	)
	// Marker must be preserved (idempotent re-formatting only).
	if strings.Count(result.Answer, "[ID:3]") < 1 {
		t.Errorf("expected [ID:3] preserved, got %q", result.Answer)
	}
}

// --- P2 helpers ---

// TestKBIDStrings_ExtractsAndFilters pins the contract of the
// KB-id-string helper used by SQL retrieval, KG retrieval, and
// DeepResearcher.
func TestKBIDStrings_ExtractsAndFilters(t *testing.T) {
	cases := []struct {
		name string
		in   []*entity.Knowledgebase
		want []string
	}{
		{"nil", nil, nil},
		{"empty", []*entity.Knowledgebase{}, nil},
		{"all empty IDs", []*entity.Knowledgebase{{ID: ""}, {ID: ""}}, nil},
		{"mixed", []*entity.Knowledgebase{{ID: "kb-1"}, nil, {ID: "kb-2"}}, []string{"kb-1", "kb-2"}},
		{"all set", []*entity.Knowledgebase{{ID: "a"}, {ID: "b"}}, []string{"a", "b"}},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			got := kbIDStrings(c.in)
			if len(got) != len(c.want) {
				t.Fatalf("kbIDStrings(%v) = %v, want %v", c.in, got, c.want)
			}
			for i := range got {
				if got[i] != c.want[i] {
					t.Errorf("kbIDStrings[%d] = %q, want %q", i, got[i], c.want[i])
				}
			}
		})
	}
}

// TestLastUserQuestion covers the helper that mirrors Python's
// `questions[-1]` access for meta_data_filter.
func TestLastUserQuestion(t *testing.T) {
	cases := []struct {
		name string
		in   []map[string]interface{}
		want string
	}{
		{"empty", nil, ""},
		{"no user", []map[string]interface{}{{"role": "system", "content": "x"}}, ""},
		{"single user", []map[string]interface{}{{"role": "user", "content": "hello"}}, "hello"},
		{"multi-turn picks last user", []map[string]interface{}{
			{"role": "user", "content": "first"},
			{"role": "assistant", "content": "ok"},
			{"role": "user", "content": "second"},
		}, "second"},
		{"non-string content", []map[string]interface{}{
			{"role": "user", "content": map[string]interface{}{"x": 1}},
		}, ""},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			if got := lastUserQuestion(c.in); got != c.want {
				t.Errorf("lastUserQuestion(%v) = %q, want %q", c.in, got, c.want)
			}
		})
	}
}

// --- P8 factory extraction test ---

// TestFactoryFromLLMID covers the helper that pulls the provider
// segment out of a composite LLMID for P8 multimodal dispatch.
func TestFactoryFromLLMID(t *testing.T) {
	cases := []struct {
		in   string
		want string
	}{
		{"", "openai"},
		{"plain-model", "openai"},
		{"qwen@local", "openai"}, // only one @ — fall back
		{"Qwen3-8B@ling@SILICONFLOW", "siliconflow"},
		{"GPT-4@openai", "openai"},
		{"claude@user@anthropic", "anthropic"},
		{"gemini-1.5@vertex@GEMINI", "gemini"},
	}
	for _, c := range cases {
		if got := factoryFromLLMID(c.in); got != c.want {
			t.Errorf("factoryFromLLMID(%q) = %q, want %q", c.in, got, c.want)
		}
	}
}

// --- P6 FullQuestion helper test ---

// TestFallbackToLatestUser pins the contract of the helper used by
// FullQuestion: when the LLM fails, we fall back to the latest user
// message content.
func TestFallbackToLatestUser(t *testing.T) {
	cases := []struct {
		name string
		in   []map[string]interface{}
		want string
	}{
		{"empty", nil, ""},
		{"no user", []map[string]interface{}{{"role": "system", "content": "x"}}, ""},
		{"single user", []map[string]interface{}{{"role": "user", "content": "hello"}}, "hello"},
		{"multi-turn picks last user", []map[string]interface{}{
			{"role": "user", "content": "first"},
			{"role": "assistant", "content": "ok"},
			{"role": "user", "content": "second"},
		}, "second"},
		{"non-string content", []map[string]interface{}{
			{"role": "user", "content": map[string]interface{}{"x": 1}},
		}, ""},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			if got := fallbackToLatestUser(c.in); got != c.want {
				t.Errorf("fallbackToLatestUser(%v) = %q, want %q", c.in, got, c.want)
			}
		})
	}
}

// --- P0/Hydration tests ---

// TestHydrateChunkVectors_NoChunksNoop pins the no-op behavior of
// the hydration helper on empty input.
func TestHydrateChunkVectors_NoChunksNoop(t *testing.T) {
	hits, err := HydrateChunkVectors(t.Context(),
		map[string]interface{}{"chunks": []interface{}{}},
		nil, nil, 0, nil,
	)
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if hits != 0 {
		t.Errorf("expected 0 hits, got %d", hits)
	}
}

// TestHydrateChunkVectors_NilKbinfosNoop pins the no-op behavior of
// the hydration helper on nil kbinfos.
func TestHydrateChunkVectors_NilKbinfosNoop(t *testing.T) {
	hits, err := HydrateChunkVectors(t.Context(), nil, nil, nil, 0, nil)
	if err != nil {
		t.Errorf("unexpected error: %v", err)
	}
	if hits != 0 {
		t.Errorf("expected 0 hits, got %d", hits)
	}
}

// TestVectorSignalHelpers pins the Python parity that a zero vector is NOT
// hydrated content: dialog_service.py:93-95 skips a chunk only when
// `any(x for x in v)`, so a placeholder of zeros must still be queued for the
// engine fetch. Without that, the agentic path — whose evidence chunks carry no
// `vector` key at all — left every chunk at zero similarity, and
// insertCitations could never cite anything, so agentic answers came back with
// no references while naive answers (real vectors inline) had them.
func TestVectorSignalHelpers(t *testing.T) {
	// No vector key at all: the agentic evidence shape.
	agentic := map[string]interface{}{"chunk_id": "c1"}
	if vectorHasSignal(chunkVector(agentic)) {
		t.Fatal("chunk without a vector must not count as hydrated")
	}
	// Zero placeholder: the naive ES shape for an unselected embedding.
	zero := map[string]interface{}{"vector": make([]float64, 4)}
	if vectorHasSignal(chunkVector(zero)) {
		t.Fatal("zero placeholder must not count as hydrated")
	}
	if got := firstChunkVectorDim([]map[string]interface{}{agentic, zero}); got != 4 {
		t.Fatalf("firstChunkVectorDim = %d, want 4 (the placeholder length names the field)", got)
	}
	// Real vector in the []interface{} shape a JSON round-trip produces.
	real := map[string]interface{}{"vector": []interface{}{0.0, 1.0, 0.0, 0.0}}
	if !vectorHasSignal(chunkVector(real)) {
		t.Fatal("non-zero vector must count as hydrated")
	}
	if got := firstChunkVectorDim([]map[string]interface{}{real}); got != 4 {
		t.Fatalf("firstChunkVectorDim = %d, want 4", got)
	}
}

// TestGetChunkValueFallsBackOnAbsenceOnly pins Python's get_value semantics
// (`d.get(k1, d.get(k2))`, rag/prompts/generator.py:37-38): the fallback fires
// on key ABSENCE only, so a key present with a nil value wins over k2.
func TestGetChunkValueFallsBackOnAbsenceOnly(t *testing.T) {
	if got := getChunkValue(map[string]interface{}{"content": nil, "content_with_weight": "body"}, "content", "content_with_weight"); got != nil {
		t.Fatalf("got %v, want nil (a present key wins even when nil)", got)
	}
	if got := getChunkValue(map[string]interface{}{"content_with_weight": "body"}, "content", "content_with_weight"); got != "body" {
		t.Fatalf("got %v, want body", got)
	}
}

// --- AsyncChatResult zero-value test ---

// TestAsyncChatResult_FinalFlagDefaultsFalse pins the zero-value
// behavior of AsyncChatResult. A non-final delta must not have
// Final=true; only the terminal result does.
func TestAsyncChatResult_FinalFlagDefaultsFalse(t *testing.T) {
	var r AsyncChatResult
	if r.Final {
		t.Errorf("zero-value AsyncChatResult should not be Final")
	}
	if r.Answer != "" {
		t.Errorf("zero-value Answer = %q, want empty", r.Answer)
	}
	if r.Prompt != "" {
		t.Errorf("zero-value Prompt = %q, want empty", r.Prompt)
	}
	if r.Reference != nil {
		t.Errorf("zero-value Reference = %v, want nil", r.Reference)
	}
}

// --- P5 SQL retrieval normalization ---

// TestNormalizeSQL_StripsThinkBlocks covers the cleanup that runs on
// the LLM-generated SQL before it's handed to the engine.
func TestNormalizeSQL_StripsThinkBlocks(t *testing.T) {
	cases := []struct {
		name string
		in   string
		want string
	}{
		{"empty", "", ""},
		{"plain", "SELECT 1", "SELECT 1"},
		{"think block", "<think>x</think>SELECT 1", "SELECT 1"},
		{"code fence", "```sql\nSELECT 1\n```", "SELECT 1"},
		{"trailing semicolon", "SELECT 1;", "SELECT 1"},
		{"all of the above", "<think>x</think>```sql\nSELECT 1;\n```", "SELECT 1"},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			if got := normalizeSQL(c.in); got != c.want {
				t.Errorf("normalizeSQL(%q) = %q, want %q", c.in, got, c.want)
			}
		})
	}
}

// TestBuildSQLReference_Scalar covers the single-row scalar shortcut in
// buildSQLReference. The path mirrors the previous
// `TestRenderSQLAnswer_Scalar` but goes through the new entry point,
// which requires constructing a minimal OpenAIChatService.
func TestBuildSQLReference_Scalar(t *testing.T) {
	s := &ChatPipelineService{}
	ans, ref := s.buildSQLReference(
		t.Context(), nil, "", "",
		[]map[string]interface{}{{"count": 42.0}},
		"", "", nil, nil, true,
	)
	if ans != "42" {
		t.Errorf("buildSQLReference scalar answer = %q, want %q", ans, "42")
	}
	// Scalar branch returns empty chunks/doc_aggs and total=1.
	if chunks, _ := ref["chunks"].([]map[string]interface{}); len(chunks) != 0 {
		t.Errorf("scalar branch chunks = %v, want empty", chunks)
	}
	if total, _ := ref["total"].(int); total != 1 {
		t.Errorf("scalar branch total = %d, want 1", total)
	}
}

// TestBuildSQLReference_MultiRowTable covers the markdown-table branch
// (multi-row, multi-column) and verifies that display columns and rows
// render correctly. Mirrors the previous
// `TestRenderSQLAnswer_MultiRowTable`.
func TestBuildSQLReference_MultiRowTable(t *testing.T) {
	rows := []map[string]interface{}{
		{"id": 1.0, "name": "alice"},
		{"id": 2.0, "name": "bob"},
	}
	s := &ChatPipelineService{}
	ans, ref := s.buildSQLReference(
		t.Context(), nil, "", "select id, name from t",
		rows,
		"sys", "elasticsearch", nil, nil, true,
	)
	// No source columns → empty chunks/doc_aggs.
	if chunks, _ := ref["chunks"].([]map[string]interface{}); len(chunks) != 0 {
		t.Errorf("non-source path chunks = %v, want empty", chunks)
	}
	if !strings.Contains(ans, "|id|") || !strings.Contains(ans, "|name|") {
		t.Errorf("expected header row, got:\n%s", ans)
	}
	if !strings.Contains(ans, "|alice|") || !strings.Contains(ans, "|bob|") {
		t.Errorf("expected data rows, got:\n%s", ans)
	}
	if !strings.Contains(ans, "|------") {
		t.Errorf("expected separator row, got:\n%s", ans)
	}
}

// --- P4 _resolve_reference_metadata ---

// TestResolveReferenceMetadata covers the prompt_config + kwargs
// resolution (matches Python's
// `resolve_reference_metadata_preferences` at
// api/utils/reference_metadata_utils.py:22-62).
func TestResolveReferenceMetadata(t *testing.T) {
	s := &ChatPipelineService{}
	cases := []struct {
		name       string
		promptCfg  map[string]interface{}
		kwargs     map[string]interface{}
		wantInc    bool
		wantFields []string
	}{
		{"all nil", nil, nil, false, nil},
		{"prompt_config only, include=false", map[string]interface{}{
			"reference_metadata": map[string]interface{}{"include": false},
		}, nil, false, nil},
		{"prompt_config only, include=true no fields", map[string]interface{}{
			"reference_metadata": map[string]interface{}{"include": true},
		}, nil, true, nil},
		{"kwargs override prompt_config", map[string]interface{}{
			"reference_metadata": map[string]interface{}{"include": true, "fields": []string{"a"}},
		}, map[string]interface{}{
			"include_metadata": false,
		}, false, nil},
		{"kwargs include_metadata true", nil, map[string]interface{}{
			"include_metadata": true,
		}, true, nil},
		{"kwargs metadata_fields only", nil, map[string]interface{}{
			"include_metadata": true,
			"metadata_fields":  []string{"author", "title"},
		}, true, []string{"author", "title"}},
		{"kwargs reference_metadata sub-dict wins", map[string]interface{}{
			"reference_metadata": map[string]interface{}{"include": true, "fields": []string{"from_config"}},
		}, map[string]interface{}{
			"reference_metadata": map[string]interface{}{"include": true, "fields": []string{"from_request"}},
		}, true, []string{"from_request"}},
		{"fields as []interface{} coerced to []string", map[string]interface{}{
			"reference_metadata": map[string]interface{}{"include": true, "fields": []interface{}{"a", "b", "c"}},
		}, nil, true, []string{"a", "b", "c"}},
	}
	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			inc, fields := s.resolveReferenceMetadata(c.promptCfg, c.kwargs)
			if inc != c.wantInc {
				t.Errorf("include = %v, want %v", inc, c.wantInc)
			}
			if !reflect.DeepEqual(fields, c.wantFields) {
				t.Errorf("fields = %v, want %v", fields, c.wantFields)
			}
		})
	}
}

// TestDecorateAnswer_VectorStrippedFromReference covers the
// reference-construction step: chunks in the Reference map have
// their `vector` field stripped (so they don't bloat the response).
func TestDecorateAnswer_VectorStrippedFromReference(t *testing.T) {
	s := &ChatPipelineService{}
	timer, _ := newTimerAndPrompt()
	kb := map[string]interface{}{
		"chunks": []map[string]interface{}{
			map[string]interface{}{
				"chunk_id":            "c1",
				"content_with_weight": "hello world",
				"vector":              []float64{0.1, 0.2, 0.3},
				"doc_id":              "d1",
			},
		},
		"doc_aggs": []interface{}{},
	}
	result := s.decorateAnswer(
		t.Context(),
		"x [ID:0]",
		kb,
		"system prompt",
		[]string{"q"},
		0,
		timer,
		nil, 0.0, true,
		nil,
		"",
		nil,
		"",
		nil,
		true,
	)
	chunks, ok := result.Reference["chunks"].([]map[string]interface{})
	if !ok || len(chunks) == 0 {
		t.Fatalf("Reference.chunks missing: %+v", result.Reference)
	}
	chunk := chunks[0]
	if _, has := chunk["vector"]; has {
		t.Errorf("vector field should be stripped from reference chunks, got %+v", chunk)
	}
}

// TestDecorateAnswer_QuoteDisabledKeepsChunks pins the cite=false shape of the
// reference: the retrieved evidence still ships, but doc_aggs does not — with
// the citation markers stripped there is nothing left to open those cards from.
func TestDecorateAnswer_QuoteDisabledKeepsChunks(t *testing.T) {
	s := &ChatPipelineService{}
	timer, _ := newTimerAndPrompt()
	kb := map[string]interface{}{
		"chunks":   []map[string]interface{}{{"chunk_id": "c1", "content_with_weight": "Evidence", "doc_id": "d1"}},
		"doc_aggs": []interface{}{map[string]interface{}{"doc_id": "d1", "doc_name": "Source"}},
	}
	result := s.decorateAnswer(t.Context(), "Answer", kb, "", nil, 0, timer,
		nil, 0, false, nil, "", nil, "", nil, true)
	if _, has := result.Reference["doc_aggs"]; has {
		t.Fatalf("disabled citations must drop doc_aggs: %#v", result.Reference)
	}
	chunks, ok := result.Reference["chunks"].([]map[string]interface{})
	if !ok || len(chunks) != 1 || chunks[0]["id"] != "c1" {
		t.Fatalf("disabled citations must still ship the evidence chunks: %#v", result.Reference["chunks"])
	}
	if result.Answer != "Answer" {
		t.Fatalf("answer changed: %q", result.Answer)
	}
}

// --- normalizeInternetFlag / shouldUseWebSearch parity with Python ---

// TestNormalizeInternetFlag_PythonParity pins the three-state return of
// the Go port against every input shape _normalize_internet_flag accepts
// in dialog_service.py:108-119. The key user-visible additions vs the
// previous Go implementation are the truthy aliases "yes" / "on" / "1"
// and the explicit falsy aliases "no" / "off" / "0" / "".
func TestNormalizeInternetFlag_PythonParity(t *testing.T) {
	tRue, fAlse := true, false
	cases := []struct {
		name string
		in   interface{}
		want *bool // nil means "couldn't interpret"
	}{
		// bool — straight through
		{"bool true", true, &tRue},
		{"bool false", false, &fAlse},

		// strings — case-insensitive, whitespace-trimmed, alias set
		{"string true", "true", &tRue},
		{"string TRUE", "TRUE", &tRue},
		{"string padded true", "  True  ", &tRue},
		{"string yes", "yes", &tRue},
		{"string on", "on", &tRue},
		{"string 1", "1", &tRue},
		{"string false", "false", &fAlse},
		{"string FALSE", "FALSE", &fAlse},
		{"string no", "no", &fAlse},
		{"string off", "off", &fAlse},
		{"string 0", "0", &fAlse},
		{"string empty", "", &fAlse},
		{"string unknown", "maybe", nil},

		// numerics — only 0 and 1 are valid (Python: `value in (0, 1)`)
		{"int 0", 0, &fAlse},
		{"int 1", 1, &tRue},
		{"int 2", 2, nil},
		{"int64 0", int64(0), &fAlse},
		{"int64 1", int64(1), &tRue},
		{"float64 0", 0.0, &fAlse},
		{"float64 1", 1.0, &tRue},
		{"float64 1.5", 1.5, nil},

		// other types → nil (couldn't interpret)
		{"nil", nil, nil},
		{"slice", []string{"true"}, nil},
		{"map", map[string]string{"a": "b"}, nil},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			got := normalizeInternetFlag(tc.in)
			switch {
			case tc.want == nil && got == nil:
				return
			case tc.want == nil && got != nil:
				t.Fatalf("input=%#v: want nil, got *%v", tc.in, *got)
			case tc.want != nil && got == nil:
				t.Fatalf("input=%#v: want *%v, got nil", tc.in, *tc.want)
			case *tc.want != *got:
				t.Fatalf("input=%#v: want *%v, got *%v", tc.in, *tc.want, *got)
			}
		})
	}
}

// TestShouldUseWebSearch_RequiresTavilyAndTruthyInternet pins the two
// conjuncts of Python's _should_use_web_search (dialog_service.py:122-126):
// tavily_api_key must be set on prompt_config AND the internet flag must
// normalize to explicit true.
func TestShouldUseWebSearch_RequiresTavilyAndTruthyInternet(t *testing.T) {
	svc := &ChatPipelineService{}
	withTavily := &entity.Chat{
		PromptConfig: entity.JSONMap{"tavily_api_key": "tvly-xxx"},
	}
	withoutTavily := &entity.Chat{
		PromptConfig: entity.JSONMap{},
	}
	nilPromptConfig := &entity.Chat{}

	cases := []struct {
		name   string
		dialog *entity.Chat
		flag   interface{}
		want   bool
	}{
		// disqualifying gates
		{"nil prompt_config", nilPromptConfig, true, false},
		{"empty tavily key", withoutTavily, true, false},
		{"tavily key + nil flag", withTavily, nil, false},
		{"tavily key + false bool", withTavily, false, false},
		{"tavily key + 'false' string", withTavily, "false", false},
		{"tavily key + unrecognized string", withTavily, "maybe", false},

		// enabling combinations — all of these were broken before
		// the normalizer fix and now work.
		{"tavily key + true bool", withTavily, true, true},
		{"tavily key + 'true' string", withTavily, "true", true},
		{"tavily key + 'yes' string (was broken)", withTavily, "yes", true},
		{"tavily key + 'on' string (was broken)", withTavily, "on", true},
		{"tavily key + '1' string (was broken)", withTavily, "1", true},
		{"tavily key + 1 int", withTavily, 1, true},
		{"tavily key + 1.0 float", withTavily, 1.0, true},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			if got := svc.shouldUseWebSearch(tc.dialog, tc.flag); got != tc.want {
				t.Fatalf("dialog=%+v flag=%#v: want %v, got %v",
					tc.dialog.PromptConfig, tc.flag, tc.want, got)
			}
		})
	}
}

// --- P5 SQL retrieval parity helpers (Python use_sql alignment) ---

// TestStripISOTimestamps verifies the dialog_service.py:1309 cleanup.
// The pattern `T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+Z)?\|` strips the
// timestamp + trailing pipe; the leading pipe/space is preserved (the
// function is meant to operate on the cell boundary). Python's
// `re.sub` has identical behavior.
func TestStripISOTimestamps(t *testing.T) {
	cases := []struct {
		name string
		in   string
		want string
	}{
		{"basic T13:24:55|", "abc T13:24:55|def", "abc |def"},
		{"with ms T13:24:55.123Z|", "abc T13:24:55.123Z|def", "abc |def"},
		{"no match", "abc|def", "abc|def"},
		{"multiple", "x T01:02:03|y T04:05:06|z", "x |y |z"},
		{"no space before T", "abcT13:24:55|def", "abc|def"},
		{"empty", "", ""},
		// Realistic Markdown cell: |2024-01-15T13:24:55| → |2024-01-15|
		{"realistic cell", "|2024-01-15T13:24:55|", "|2024-01-15|"},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			if got := stripISOTimestamps(tc.in); got != tc.want {
				t.Errorf("stripISOTimestamps(%q) = %q, want %q", tc.in, got, tc.want)
			}
		})
	}
}

// TestMapColumnName exercises the map_column_name algorithm at
// dialog_service.py:1238-1280.
func TestMapColumnName(t *testing.T) {
	fieldMap := map[string]interface{}{
		"title":      "Title",
		"issue_date": "Issue Date (/Day/Month/Year)",
		"docnm":      "Document Name",
		"docnm_kwd":  "Document Name",
	}
	cases := []struct {
		name string
		col  string
		fm   map[string]interface{}
		want string
	}{
		{"count(star) special case", "count(star)", nil, "COUNT(*)"},
		{"count(star) case-insensitive", "COUNT(STAR)", nil, "COUNT(*)"},
		{"AS alias in field_map", "json_extract_string(c, '$.title') AS title", fieldMap, "Title"},
		{"AS alias not in field_map, case-insensitive", "fn() AS TITLE", fieldMap, "Title"},
		{"AS alias unknown, return as-is", "fn() AS unknown_alias", fieldMap, "unknown_alias"},
		{"direct match", "title", fieldMap, "Title"},
		{"direct case-insensitive", "TITLE", fieldMap, "Title"},
		{"no match, bulk replace", "json_extract_string(c, '$.title')", fieldMap, "json_extract_string(c, '$.Title')"},
		// `(/.*|...)` matches "/Day/Month/Year)" and replaces with "".
		// The leading `(` is left intact — this matches Python's
		// `re.sub` behavior exactly.
		{"paren suffix stripped", "issue_date", fieldMap, "Issue Date ("},
		{"empty field map returns alias", "fn() AS foo", map[string]interface{}{}, "foo"},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			fm := tc.fm
			if fm == nil && tc.name == "count(star) special case" || tc.name == "count(star) case-insensitive" {
				fm = map[string]interface{}{}
			}
			if got := mapColumnName(tc.col, fm); got != tc.want {
				t.Errorf("mapColumnName(%q) = %q, want %q", tc.col, got, tc.want)
			}
		})
	}
}

// TestChunkKBIDForDoc mirrors _chunk_kb_id_for_doc at dialog_service.py:56-59.
func TestChunkKBIDForDoc(t *testing.T) {
	cases := []struct {
		name    string
		rowDict map[string]interface{}
		kbIDs   []string
		docID   interface{}
		want    string
	}{
		{
			name:    "single kb returns kbIDs[0]",
			rowDict: map[string]interface{}{},
			kbIDs:   []string{"kb_a"},
			docID:   "doc1",
			want:    "kb_a",
		},
		{
			name:    "multi kb with kb_id in row",
			rowDict: map[string]interface{}{"kb_id": "kb_b"},
			kbIDs:   []string{"kb_a", "kb_b"},
			docID:   "doc1",
			want:    "kb_b",
		},
		{
			name:    "multi kb with kb_id_kwd in row (no kb_id)",
			rowDict: map[string]interface{}{"kb_id_kwd": "kb_c"},
			kbIDs:   []string{"kb_a", "kb_b"},
			docID:   "doc1",
			want:    "kb_c",
		},
		{
			name:    "multi kb with neither returns empty",
			rowDict: map[string]interface{}{},
			kbIDs:   []string{"kb_a", "kb_b"},
			docID:   "doc1",
			want:    "",
		},
		{
			name:    "multi kb with empty kb_id falls through to kb_id_kwd",
			rowDict: map[string]interface{}{"kb_id": "", "kb_id_kwd": "kb_d"},
			kbIDs:   []string{"kb_a", "kb_b"},
			docID:   "doc1",
			want:    "kb_d",
		},
		{
			name:    "no kbIDs falls through to row lookup",
			rowDict: map[string]interface{}{"kb_id": "kb_a"},
			kbIDs:   nil,
			docID:   "doc1",
			want:    "kb_a",
		},
		{
			name:    "no kbIDs and no row kb_id returns empty",
			rowDict: map[string]interface{}{},
			kbIDs:   nil,
			docID:   "doc1",
			want:    "",
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			if got := chunkKBIDForDoc(tc.rowDict, tc.kbIDs, tc.docID); got != tc.want {
				t.Errorf("chunkKBIDForDoc = %q, want %q", got, tc.want)
			}
		})
	}
}

// TestCleanCellValue verifies the per-cell rendering at
// dialog_service.py:1298 (remove_redundant_spaces + replace None with space).
func TestCleanCellValue(t *testing.T) {
	cases := []struct {
		name string
		in   interface{}
		want string
	}{
		{"string", "hello", "hello"},
		{"float", 42.0, "42"},
		{"int", 42, "42"},
		{"None string literal", "None", " "},
		{"string with redundant space after (", "( world", "(world"},
		{"empty", "", ""},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			if got := cleanCellValue(tc.in); got != tc.want {
				t.Errorf("cleanCellValue(%v) = %q, want %q", tc.in, got, tc.want)
			}
		})
	}
}

// TestExtractSourceColumnIndexes verifies stable sorted column ordering
// and case-insensitive source-column detection.
func TestExtractSourceColumnIndexes(t *testing.T) {
	rows := []map[string]interface{}{
		{"DOC_ID": "d1", "docnm_kwd": "Doc1", "title": "T1", "kb_id": "k1"},
	}
	docIDIdx, docNameIdx, kbIDIdx, columns := extractSourceColumnIndexes(rows)
	if len(docIDIdx) != 1 {
		t.Errorf("expected 1 doc_id index, got %d", len(docIDIdx))
	}
	if len(docNameIdx) != 1 {
		t.Errorf("expected 1 doc_name index, got %d", len(docNameIdx))
	}
	if len(kbIDIdx) != 1 {
		t.Errorf("expected 1 kb_id index, got %d", len(kbIDIdx))
	}
	// Columns must be sorted alphabetically: DOC_ID, docnm_kwd, kb_id, title
	wantCols := []string{"DOC_ID", "docnm_kwd", "kb_id", "title"}
	if !reflect.DeepEqual(columns, wantCols) {
		t.Errorf("columns = %v, want %v", columns, wantCols)
	}
	// Empty rows returns empty slices.
	emptyDocID, _, _, _ := extractSourceColumnIndexes(nil)
	if len(emptyDocID) != 0 {
		t.Errorf("empty rows docIDIdx = %v, want empty", emptyDocID)
	}
}

// TestBuildChunkFetchSQL verifies the WHERE-clause extraction and SQL
// construction at dialog_service.py:1321-1331.
func TestBuildChunkFetchSQL(t *testing.T) {
	cases := []struct {
		name      string
		sql       string
		multiKB   bool
		wantSQL   string
		wantFound bool
	}{
		{
			name:      "WHERE + GROUP BY (extracts up to GROUP BY)",
			sql:       "select count(*) from t where x = 1 group by y",
			multiKB:   false,
			wantSQL:   "select doc_id, docnm_kwd from t where x = 1 limit 20",
			wantFound: true,
		},
		{
			name:      "WHERE only, single KB, no limit",
			sql:       "select * from t where x = 1",
			multiKB:   false,
			wantSQL:   "select doc_id, docnm_kwd from t where x = 1 limit 20",
			wantFound: true,
		},
		{
			name:      "WHERE only, multi KB adds kb_id column",
			sql:       "select * from t where x = 1",
			multiKB:   true,
			wantSQL:   "select doc_id, docnm_kwd, kb_id from t where x = 1 limit 20",
			wantFound: true,
		},
		{
			// The secondary fetch drops the aggregate query's trailing clauses
			// and applies its own limit.
			name:      "WHERE + ORDER BY + LIMIT 5 (suffixes dropped, no extra limit)",
			sql:       "select * from t where x = 1 order by y limit 5",
			multiKB:   false,
			wantSQL:   "select doc_id, docnm_kwd from t where x = 1 limit 20",
			wantFound: true,
		},
		{
			name:      "no WHERE returns not-found",
			sql:       "select * from t",
			multiKB:   false,
			wantSQL:   "",
			wantFound: false,
		},
		{
			// Python's f-string emits a literal lowercase "where";
			// the original case from the input is NOT preserved.
			name:      "case-insensitive where (output uses lowercase where)",
			sql:       "select * from t WHERE x = 1",
			multiKB:   false,
			wantSQL:   "select doc_id, docnm_kwd from t where x = 1 limit 20",
			wantFound: true,
		},
		{
			name:      "Infinity expectedCol is docnm (not _kwd)",
			sql:       "select * from t where x = 1",
			multiKB:   false,
			wantSQL:   "select doc_id, docnm from t where x = 1 limit 20",
			wantFound: true,
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			expectedCol := "docnm_kwd"
			if tc.name == "Infinity expectedCol is docnm (not _kwd)" {
				expectedCol = "docnm"
			}
			gotSQL, gotFound := buildChunkFetchSQL(tc.sql, "t", expectedCol, tc.multiKB, "elasticsearch")
			if gotFound != tc.wantFound {
				t.Errorf("found = %v, want %v", gotFound, tc.wantFound)
			}
			if gotSQL != tc.wantSQL {
				t.Errorf("sql = %q, want %q", gotSQL, tc.wantSQL)
			}
		})
	}
}

// TestToIfaceSlice verifies the slice type conversion for the call-site
// contract at chat_pipeline.go:3846.
func TestToIfaceSlice(t *testing.T) {
	in := []map[string]interface{}{
		{"a": 1},
		{"b": 2},
	}
	out := toIfaceSlice(in)
	if len(out) != 2 {
		t.Fatalf("len = %d, want 2", len(out))
	}
	if _, ok := out[0].(map[string]interface{}); !ok {
		t.Errorf("element 0 type = %T, want map[string]interface{}", out[0])
	}
}

// TestExpectedDocNameColumn verifies the engine→column name mapping.
func TestExpectedDocNameColumn(t *testing.T) {
	if got := expectedDocNameColumn("infinity"); got != "docnm" {
		t.Errorf("infinity = %q, want docnm", got)
	}
	if got := expectedDocNameColumn("oceanbase"); got != "docnm_kwd" {
		t.Errorf("oceanbase = %q, want docnm_kwd", got)
	}
	if got := expectedDocNameColumn("seekdb"); got != "docnm_kwd" {
		t.Errorf("seekdb = %q, want docnm_kwd", got)
	}
	if got := expectedDocNameColumn("vastbase"); got != "docnm_kwd" {
		t.Errorf("vastbase = %q, want docnm_kwd", got)
	}
	if got := expectedDocNameColumn("elasticsearch"); got != "docnm_kwd" {
		t.Errorf("elasticsearch = %q, want docnm_kwd", got)
	}
	if got := expectedDocNameColumn("opensearch"); got != "docnm_kwd" {
		t.Errorf("opensearch = %q, want docnm_kwd", got)
	}
	if got := expectedDocNameColumn("unknown"); got != "docnm_kwd" {
		t.Errorf("unknown = %q, want docnm_kwd", got)
	}
}

// TestIsAggregateSQL matches the regex from dialog_service.py:974.
func TestIsAggregateSQL(t *testing.T) {
	cases := []struct {
		sql  string
		want bool
	}{
		{"select count(*) from t", true},
		{"select sum(x) from t", true},
		{"select avg(x) from t", true},
		{"select max(x), min(y) from t", true},
		{"select count(distinct x) from t", true},
		{"select * from t where x = 1", false},
		{"select distinct x from t", false}, // bare DISTINCT without ( ) doesn't match
		{"", false},
	}
	for _, tc := range cases {
		t.Run(tc.sql, func(t *testing.T) {
			if got := isAggregateSQL(tc.sql); got != tc.want {
				t.Errorf("isAggregateSQL(%q) = %v, want %v", tc.sql, got, tc.want)
			}
		})
	}
}

// sqlFakeEngine is a minimal in-memory engine.DocEngine stub for
// testing fetchAggregateChunks / buildSQLReference without a real
// engine. It embeds engine.DocEngine to satisfy the interface (the
// embedded methods will panic if accidentally called, which is the
// intended loud-failure mode).
type sqlFakeEngine struct {
	engine.DocEngine
	engineType string
	sqlCalls   *[]string
	rowsBySQL  map[string][]map[string]interface{}
	errBySQL   map[string]error
	runSQL     func(ctx context.Context, sqlText string) ([]map[string]interface{}, error)
}

func (f *sqlFakeEngine) GetType() string { return f.engineType }
func (f *sqlFakeEngine) RunSQL(ctx context.Context, sqlText string, format string) ([]map[string]interface{}, error) {
	if f.runSQL != nil {
		return f.runSQL(ctx, sqlText)
	}
	if f.sqlCalls != nil {
		*f.sqlCalls = append(*f.sqlCalls, sqlText)
	}
	if f.errBySQL != nil {
		if err, ok := f.errBySQL[sqlText]; ok {
			return nil, err
		}
	}
	if f.rowsBySQL != nil {
		if rows, ok := f.rowsBySQL[sqlText]; ok {
			return rows, nil
		}
	}
	return nil, nil
}

// TestFetchAggregateChunks_SkipsInfinityMultiKB verifies the
// Infinity multi-KB short-circuit (mirrors Python's add_kb_filter
// no-op for Infinity).
func TestFetchAggregateChunks_SkipsInfinityMultiKB(t *testing.T) {
	sqlEngine := &sqlFakeEngine{engineType: "infinity"}
	s := &ChatPipelineService{}
	chunks, docAggs := s.fetchAggregateChunks(
		t.Context(), sqlEngine, "t",
		"select count(*) from t where x = 1",
		"docnm", []string{"kb_a", "kb_b"},
	)
	if chunks != nil || docAggs != nil {
		t.Errorf("expected nil chunks/docAggs on Infinity multi-KB, got %v / %v", chunks, docAggs)
	}
}

// TestFetchAggregateChunks_SingleKBSuccess verifies the secondary fetch
// path populates chunks and doc_aggs correctly.
func TestFetchAggregateChunks_SingleKBSuccess(t *testing.T) {
	chunksSQL := "select doc_id, docnm_kwd from t WHERE (x = 1) AND (kb_id = '" + scopeAllowedKB + "') limit 20"
	sqlEngine := &sqlFakeEngine{
		engineType: "elasticsearch",
		rowsBySQL: map[string][]map[string]interface{}{
			chunksSQL: {
				{"doc_id": "d1", "docnm_kwd": "Doc1"},
				{"doc_id": "d2", "docnm_kwd": "Doc2"},
				{"doc_id": "d1", "docnm_kwd": "Doc1"},
			},
		},
	}
	s := &ChatPipelineService{}
	chunks, docAggs := s.fetchAggregateChunks(
		t.Context(), sqlEngine, "t",
		"select count(*) from t where x = 1",
		"docnm_kwd", []string{scopeAllowedKB},
	)
	if len(chunks) != 3 {
		t.Fatalf("chunks len = %d, want 3", len(chunks))
	}
	if len(docAggs) != 2 {
		t.Fatalf("docAggs len = %d, want 2", len(docAggs))
	}
	// d1 appears twice → count=2; d2 once → count=1.
	counts := map[string]int{}
	for _, agg := range docAggs {
		counts[agg["doc_id"].(string)] = agg["count"].(int)
	}
	if counts["d1"] != 2 || counts["d2"] != 1 {
		t.Errorf("counts = %v, want d1=2, d2=1", counts)
	}
	// Single-kb: each chunk gets kb_id from the dialog's kb list.
	for i, c := range chunks {
		if c["kb_id"] != scopeAllowedKB {
			t.Errorf("chunks[%d].kb_id = %v, want %s", i, c["kb_id"], scopeAllowedKB)
		}
	}
}

// TestFetchAggregateChunks_NoWhereClause verifies the no-WHERE early
// return (matches Python's aggregate fallback at L1365).
func TestFetchAggregateChunks_NoWhereClause(t *testing.T) {
	sqlEngine := &sqlFakeEngine{engineType: "elasticsearch"}
	s := &ChatPipelineService{}
	chunks, docAggs := s.fetchAggregateChunks(
		t.Context(), sqlEngine, "t",
		"select count(*) from t",
		"docnm_kwd", []string{"kb_a"},
	)
	if chunks != nil || docAggs != nil {
		t.Errorf("expected nil on no-WHERE, got %v / %v", chunks, docAggs)
	}
}

// TestFetchAggregateChunks_RunSQLError verifies graceful failure.
func TestFetchAggregateChunks_RunSQLError(t *testing.T) {
	called := false
	sqlEngine := &sqlFakeEngine{
		engineType: "elasticsearch",
		runSQL: func(ctx context.Context, sqlText string) ([]map[string]interface{}, error) {
			called = true
			return nil, fmt.Errorf("engine boom")
		},
	}
	s := &ChatPipelineService{}
	chunks, docAggs := s.fetchAggregateChunks(
		t.Context(), sqlEngine, "t",
		"select count(*) from t where x = 1",
		"docnm_kwd", []string{scopeAllowedKB},
	)
	if !called {
		t.Fatal("expected the secondary query to execute")
	}
	if chunks != nil || docAggs != nil {
		t.Errorf("expected nil on RunSQL error, got %v / %v", chunks, docAggs)
	}
}

// TestBuildSQLReference_EmptyRows verifies the empty-rows path.
func TestBuildSQLReference_EmptyRows(t *testing.T) {
	s := &ChatPipelineService{}
	ans, ref := s.buildSQLReference(
		t.Context(), nil, "", "", nil,
		"", "", nil, nil, true,
	)
	if ans != "No results." {
		t.Errorf("ans = %q, want %q", ans, "No results.")
	}
	if total, _ := ref["total"].(int); total != 0 {
		t.Errorf("total = %d, want 0", total)
	}
}

// TestBuildSQLReference_NonAggregateWithSourceColumns verifies that
// chunks and doc_aggs are populated from rows when source columns
// are present.
func TestBuildSQLReference_NonAggregateWithSourceColumns(t *testing.T) {
	rows := []map[string]interface{}{
		{"doc_id": "d1", "docnm_kwd": "Doc1", "title": "T1"},
		{"doc_id": "d2", "docnm_kwd": "Doc2", "title": "T2"},
	}
	kbs := []*entity.Knowledgebase{{ID: "kb_a"}}
	s := &ChatPipelineService{}
	ans, ref := s.buildSQLReference(
		t.Context(), nil, "t", "select doc_id, docnm_kwd, title from t",
		rows, "", "elasticsearch", kbs, nil, true,
	)
	if !strings.Contains(ans, "Source|") {
		t.Errorf("expected Source column in answer, got:\n%s", ans)
	}
	if !strings.Contains(ans, "##0$$") || !strings.Contains(ans, "##1$$") {
		t.Errorf("expected ##N$$ citation markers, got:\n%s", ans)
	}
	chunks, _ := ref["chunks"].([]map[string]interface{})
	if len(chunks) != 2 {
		t.Fatalf("chunks len = %d, want 2", len(chunks))
	}
	docAggs, _ := ref["doc_aggs"].([]map[string]interface{})
	if len(docAggs) != 2 {
		t.Fatalf("docAggs len = %d, want 2", len(docAggs))
	}
	// Each chunk must carry dataset_id (remapped from kb_id by chunksFormat) from the single-KB dialog.
	for i, cm := range chunks {
		if cm["dataset_id"] != "kb_a" {
			t.Errorf("chunks[%d].dataset_id = %v, want kb_a", i, cm["dataset_id"])
		}
	}

	ans, ref = s.buildSQLReference(
		t.Context(), nil, "t", "select doc_id, docnm_kwd, title from t",
		rows, "", "elasticsearch", kbs, nil, false,
	)
	if strings.Contains(ans, "Source|") || strings.Contains(ans, "##0$$") {
		t.Errorf("quote=false leaked SQL citations in answer:\n%s", ans)
	}
	if len(ref) != 0 {
		t.Errorf("quote=false reference = %#v, want empty", ref)
	}
}

// TestBuildSQLReference_AggregateMissingSourceColumnsSecondaryFetch
// verifies that an aggregate SQL with no source columns triggers the
// secondary fetch and uses its result for chunks/doc_aggs.
//
// The test uses a multi-cell aggregate (1 row, 2 columns) to avoid the
// scalar shortcut at the top of buildSQLReference.
func TestBuildSQLReference_AggregateMissingSourceColumnsSecondaryFetch(t *testing.T) {
	rows := []map[string]interface{}{
		{"count": 42.0, "label": "total"},
	}
	chunksSQL := "select doc_id, docnm_kwd from t WHERE (x = 1) AND (kb_id = '" + scopeAllowedKB + "') limit 20"
	sqlEngine := &sqlFakeEngine{
		engineType: "elasticsearch",
		rowsBySQL: map[string][]map[string]interface{}{
			chunksSQL: {
				{"doc_id": "d1", "docnm_kwd": "Doc1"},
			},
		},
	}
	kbs := []*entity.Knowledgebase{{ID: scopeAllowedKB}}
	s := &ChatPipelineService{}
	ans, ref := s.buildSQLReference(
		t.Context(), sqlEngine, "t",
		"select count(*) from t where x = 1",
		rows, "", "elasticsearch", kbs, nil, true,
	)
	// Multi-cell aggregate → renders as a table, not a scalar.
	if !strings.Contains(ans, "|42|") {
		t.Errorf("ans = %q, want to contain |42|", ans)
	}
	chunks, _ := ref["chunks"].([]map[string]interface{})
	if len(chunks) != 1 {
		t.Errorf("chunks len = %d, want 1 (from secondary fetch)", len(chunks))
	}
}

// TestBuildSQLReference_NonAggregateMissingSourceEmptyRefs verifies
// that non-aggregate SQL without source columns returns the table but
// empty chunks/doc_aggs (Python's best-effort path at L1367).
func TestBuildSQLReference_NonAggregateMissingSourceEmptyRefs(t *testing.T) {
	rows := []map[string]interface{}{
		{"title": "T1"},
		{"title": "T2"},
	}
	s := &ChatPipelineService{}
	ans, ref := s.buildSQLReference(
		t.Context(), nil, "t", "select title from t",
		rows, "", "elasticsearch", nil, nil, true,
	)
	if !strings.Contains(ans, "T1") || !strings.Contains(ans, "T2") {
		t.Errorf("expected table data in answer, got:\n%s", ans)
	}
	if strings.Contains(ans, "Source|") {
		t.Errorf("expected no Source column, got:\n%s", ans)
	}
	chunks, _ := ref["chunks"].([]map[string]interface{})
	if len(chunks) != 0 {
		t.Errorf("chunks = %v, want empty", chunks)
	}
	docAggs, _ := ref["doc_aggs"].([]interface{})
	if len(docAggs) != 0 {
		t.Errorf("docAggs = %v, want empty", docAggs)
	}
}

// TestBuildSQLReference_DisplayNameTranslation verifies that column
// names are translated via the field_map.
func TestBuildSQLReference_DisplayNameTranslation(t *testing.T) {
	rows := []map[string]interface{}{
		{"doc_id": "d1", "docnm_kwd": "Doc1", "title": "Hello"},
	}
	fieldMap := map[string]interface{}{"title": "My Title"}
	s := &ChatPipelineService{}
	ans, _ := s.buildSQLReference(
		t.Context(), nil, "t", "select doc_id, docnm_kwd, title from t",
		rows, "", "elasticsearch", nil, fieldMap, true,
	)
	if !strings.Contains(ans, "|My Title|") {
		t.Errorf("expected translated column name, got:\n%s", ans)
	}
	if strings.Contains(ans, "|title|") {
		t.Errorf("raw column name should not appear, got:\n%s", ans)
	}
}

// TestBuildSQLReference_ISOTimestampStripped verifies that ISO
// timestamps in cell values are stripped from the rendered table.
func TestBuildSQLReference_ISOTimestampStripped(t *testing.T) {
	rows := []map[string]interface{}{
		{"doc_id": "d1", "docnm_kwd": "Doc1", "created_at": "2024-01-15T13:24:55"},
	}
	s := &ChatPipelineService{}
	ans, _ := s.buildSQLReference(
		t.Context(), nil, "t", "select doc_id, docnm_kwd, created_at from t",
		rows, "", "elasticsearch", nil, nil, true,
	)
	if strings.Contains(ans, "T13:24:55") {
		t.Errorf("expected ISO timestamp stripped, got:\n%s", ans)
	}
	if !strings.Contains(ans, "2024-01-15") {
		t.Errorf("expected date portion preserved, got:\n%s", ans)
	}
}

// --- BuildChatConfig unit tests (moved from openai_chat_test.go) ---

// TestBuildChatConfig_RequestOverrides pins down the merge order:
// dialog.LLMSetting is the base; request fields override.
func TestBuildChatConfig_RequestOverrides(t *testing.T) {
	temp := 0.1
	dialog := &entity.Chat{
		LLMSetting: entity.JSONMap{
			"temperature": 0.5,
			"top_p":       0.9,
			"max_tokens":  float64(512),
		},
	}
	req := map[string]interface{}{
		"temperature": temp,
		"max_tokens":  128,
	}
	cfg := BuildChatConfig(dialog, req)
	if cfg.Temperature == nil || *cfg.Temperature != temp {
		t.Fatalf("expected request temperature %v, got %v", temp, cfg.Temperature)
	}
	if cfg.TopP == nil || *cfg.TopP != 0.9 {
		t.Fatalf("expected dialog top_p 0.9 to be preserved, got %v", cfg.TopP)
	}
	if cfg.MaxTokens == nil || *cfg.MaxTokens != 128 {
		t.Fatalf("expected request max_tokens 128, got %v", cfg.MaxTokens)
	}
}

// TestBuildChatConfig_FromEmptyDialog verifies the merger works even when
// dialog.LLMSetting is nil.
func TestBuildChatConfig_FromEmptyDialog(t *testing.T) {
	temp := 0.3
	dialog := &entity.Chat{}
	req := map[string]interface{}{"temperature": temp}
	cfg := BuildChatConfig(dialog, req)
	if cfg.Temperature == nil || *cfg.Temperature != temp {
		t.Fatalf("expected temperature %v, got %v", temp, cfg.Temperature)
	}
}

func TestClampChatConfigMaxTokensUsesRemainingBudget(t *testing.T) {
	dialog := &entity.Chat{
		LLMSetting: entity.JSONMap{"max_tokens": float64(1024)},
	}
	cfg := BuildChatConfig(dialog, map[string]interface{}{"max_tokens": 700})

	adjusted, ok, err := clampChatConfigMaxTokens(cfg, 1000, 450)
	if err != nil {
		t.Fatalf("clampChatConfigMaxTokens returned error: %v", err)
	}
	if !ok {
		t.Fatal("expected max_tokens to be clamped")
	}
	if adjusted != 550 {
		t.Fatalf("adjusted max_tokens = %d, want 550", adjusted)
	}
	if cfg.MaxTokens == nil || *cfg.MaxTokens != 550 {
		t.Fatalf("config max_tokens = %v, want 550", cfg.MaxTokens)
	}
	if dialog.LLMSetting["max_tokens"] != float64(1024) {
		t.Fatalf("dialog max_tokens was mutated: %v", dialog.LLMSetting["max_tokens"])
	}
}

func TestBuildChatConfigIgnoresNonPositiveMaxTokens(t *testing.T) {
	dialog := &entity.Chat{
		LLMSetting: entity.JSONMap{"max_tokens": float64(0)},
	}
	cfg := BuildChatConfig(dialog, nil)
	if cfg.MaxTokens != nil {
		t.Fatalf("persisted max_tokens=0 must not populate ChatConfig, got %v", *cfg.MaxTokens)
	}

	cfg = BuildChatConfig(&entity.Chat{}, map[string]interface{}{"max_tokens": -1})
	if cfg.MaxTokens != nil {
		t.Fatalf("request max_tokens=-1 must not populate ChatConfig, got %v", *cfg.MaxTokens)
	}
}

func TestClampChatConfigMaxTokensRejectsNonPositive(t *testing.T) {
	for _, value := range []int{0, -1} {
		cfg := &modelModule.ChatConfig{MaxTokens: &value}
		if adjusted, ok, err := clampChatConfigMaxTokens(cfg, 1000, 100); err != nil {
			t.Fatalf("max_tokens=%d returned unexpected error: %v", value, err)
		} else if ok {
			t.Fatalf("max_tokens=%d unexpectedly clamped to %d", value, adjusted)
		}
		if cfg.MaxTokens != nil {
			t.Fatalf("max_tokens=%d should be cleared before provider request, got %v", value, *cfg.MaxTokens)
		}
	}
}

func TestClampChatConfigMaxTokensRejectsExhaustedCapacity(t *testing.T) {
	for _, usedTokenCount := range []int{1000, 1001} {
		maxTokens := 700
		cfg := &modelModule.ChatConfig{MaxTokens: &maxTokens}

		if adjusted, ok, err := clampChatConfigMaxTokens(cfg, 1000, usedTokenCount); err == nil {
			t.Fatalf("usedTokenCount=%d expected capacity error, got adjusted=%d ok=%t", usedTokenCount, adjusted, ok)
		}
		if cfg.MaxTokens == nil || *cfg.MaxTokens != maxTokens {
			t.Fatalf("capacity error should not set max_tokens to zero, got %v", cfg.MaxTokens)
		}
	}
}

// collectSink records every delta with its isThink flag.
func collectSink(got *[]string, thinks *[]bool) func(string, bool) {
	return func(delta string, isThink bool) {
		*got = append(*got, delta)
		*thinks = append(*thinks, isThink)
	}
}

func TestAsyncChatEmptyResponseQuoteDisabled(t *testing.T) {
	db := setupChatPipelineVisionTestDB(t)
	seedChatPipelineVisionTenant(t, db, "glm-4-flash@ZHIPU-AI")
	if err := db.AutoMigrate(&entity.Knowledgebase{}); err != nil {
		t.Fatal(err)
	}
	if err := db.Create(&entity.Knowledgebase{ID: "kb-1", TenantID: "tenant-1", CreatedBy: "user-1", Name: "Knowledge"}).Error; err != nil {
		t.Fatal(err)
	}

	chat := dialForTest("glm-4-flash@ZHIPU-AI")
	chat.KBIDs = entity.JSONSlice{"kb-1"}
	chat.PromptConfig["system"] = "Answer using {knowledge}"
	chat.PromptConfig["empty_response"] = "Nothing found."
	ctx := serviceTestContext(t, "user-1", "tenant-1")
	results, err := NewChatPipelineService().AsyncChat(ctx, chat,
		[]map[string]interface{}{{"role": "user", "content": "Question"}}, true,
		map[string]interface{}{"quote": false})
	if err != nil {
		t.Fatal(err)
	}

	var final AsyncChatResult
	for result := range results {
		if result.Final {
			final = result
		}
	}
	if !final.Final || final.Answer != "Nothing found." {
		t.Fatalf("final=%+v, want empty-response result", final)
	}
	if final.Reference == nil || len(final.Reference) != 0 {
		t.Fatalf("quote=false reference = %#v, want explicit empty object", final.Reference)
	}
}

// TestThinkEventWireKeys pins the JSON keys of a fully populated step: they ARE
// the client contract (the SSE `think_event` field), so renaming or dropping a tag
// breaks every client while neither the compiler nor any Go caller notices.
//
// It is also the only assertion on this struct's shape, which is why a producer
// ALIASES this type instead of carrying a second copy bridged field by field —
// that copy compiled for as long as it stayed a subset, and silently dropped
// any field added on the other side.
func TestThinkEventWireKeys(t *testing.T) {
	// Every field populated: with `omitempty` on all but Kind and Summary, that is
	// what makes the whole key set appear.
	full := ThinkEvent{
		Kind:       "tool_result",
		Stage:      "Function tool",
		Tool:       "retrieve",
		Args:       `{"query":"q"}`,
		Status:     "ok",
		Reason:     "no_doc",
		Cause:      "the knowledge base returned no searchable hits",
		Results:    3,
		Documents:  2,
		Sources:    []string{"c1", "c2"},
		DurationMS: 12,
		Summary:    "The retrieve tool returned 3 results from 2 documents.",
	}
	want := []string{
		"args", "cause", "documents", "duration_ms", "kind", "reason",
		"results", "sources", "stage", "status", "summary", "tool",
	}
	if got := thinkEventWireKeys(t, full); strings.Join(got, ",") != strings.Join(want, ",") {
		t.Errorf("wire keys = %v, want %v", got, want)
	}

	// A stage step carries only Kind, Stage and Summary; everything else is
	// omitted rather than sent as a zero — a client must treat each optional field
	// as absent-when-zero, not as present-and-empty.
	got := thinkEventWireKeys(t, ThinkEvent{Kind: "stage", Stage: "RAGAgent", Summary: "x"})
	if strings.Join(got, ",") != "kind,stage,summary" {
		t.Errorf("stage step keys = %v, want kind,stage,summary", got)
	}
}

// thinkEventWireKeys marshals one event and returns its sorted JSON key set.
func thinkEventWireKeys(t *testing.T, ev ThinkEvent) []string {
	t.Helper()
	raw, err := json.Marshal(ev)
	if err != nil {
		t.Fatalf("marshal: %v", err)
	}
	var decoded map[string]json.RawMessage
	if err := json.Unmarshal(raw, &decoded); err != nil {
		t.Fatalf("unmarshal %s: %v", raw, err)
	}
	keys := make([]string, 0, len(decoded))
	for k := range decoded {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	return keys
}

// setupChatPipelineToolSupportTestDB seeds an in-memory scope whose models differ
// only in their persisted tool-calling flag, so the probe runs without the provider
// catalog:
//
//	model-tools-on        — chat,    extra {"is_tools": true}
//	model-tools-off       — chat,    extra {"is_tools": false}
//	model-tools-disabled  — chat,    extra {"is_tools": true}, status inactive
//	model-embedding-tools — embedding, extra {"is_tools": true}
func setupChatPipelineToolSupportTestDB(t *testing.T) *gorm.DB {
	t.Helper()

	db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{TranslateError: true})
	if err != nil {
		t.Fatalf("failed to open sqlite: %v", err)
	}
	if err := db.AutoMigrate(
		&entity.Tenant{},
		&entity.UserTenant{},
		&entity.TenantModelProvider{},
		&entity.TenantModelInstance{},
		&entity.TenantModel{},
	); err != nil {
		t.Fatalf("failed to migrate model tables: %v", err)
	}

	orig := dao.DB
	dao.DB = db
	t.Cleanup(func() { dao.DB = orig })

	activeStatus := "1"
	rows := []interface{}{
		&entity.UserTenant{ID: "user-tenant-1", UserID: "user-1", TenantID: "tenant-1", Role: "owner", InvitedBy: "user-1", Status: &activeStatus},
		&entity.TenantModelProvider{ID: "provider-zhipu", UserID: "user-1", TenantID: "tenant-1", ProviderName: "ZHIPU-AI"},
		&entity.TenantModelInstance{ID: "instance-zhipu", ProviderID: "provider-zhipu", InstanceName: "default", APIKey: "sk-test", Status: "active", Extra: "{}"},
		&entity.TenantModel{ID: "model-tools-on", ProviderID: "provider-zhipu", InstanceID: "instance-zhipu", ModelName: "glm-4-plus", ModelType: int(entity.ModelTypeChat), Status: "active", Extra: `{"is_tools":true}`},
		&entity.TenantModel{ID: "model-tools-off", ProviderID: "provider-zhipu", InstanceID: "instance-zhipu", ModelName: "glm-4-flash", ModelType: int(entity.ModelTypeChat), Status: "active", Extra: `{"is_tools":false}`},
		&entity.TenantModel{ID: "model-tools-disabled", ProviderID: "provider-zhipu", InstanceID: "instance-zhipu", ModelName: "glm-4-air", ModelType: int(entity.ModelTypeChat), Status: "inactive", Extra: `{"is_tools":true}`},
		&entity.TenantModel{ID: "model-embedding-tools", ProviderID: "provider-zhipu", InstanceID: "instance-zhipu", ModelName: "embedding-3", ModelType: int(entity.ModelTypeEmbedding), Status: "active", Extra: `{"is_tools":true}`},
		// Enrolled ONLY as image-to-text: a valid chat-pipeline input, and the case
		// that used to be resolved as chat and rejected.
		&entity.TenantModel{ID: "model-image2text-tools", ProviderID: "provider-zhipu", InstanceID: "instance-zhipu", ModelName: "glm-4v-plus", ModelType: int(entity.ModelTypeImage2Text), Status: "active", Extra: `{"is_tools":true}`},
	}
	for _, row := range rows {
		if err := db.Create(row).Error; err != nil {
			t.Fatalf("failed to seed %T: %v", row, err)
		}
	}
	return db
}

// seedChatPipelineToolSupportTenant sets the owner membership's default chat model, which
// the no-explicit-LLM branch of the probe resolves.
func seedChatPipelineToolSupportTenant(t *testing.T, db *gorm.DB, defaultLLMID string) {
	t.Helper()
	status := string(entity.StatusValid)
	if err := db.Create(&entity.Tenant{
		ID:     "tenant-1",
		Status: &status,
	}).Error; err != nil {
		t.Fatalf("failed to create tenant: %v", err)
	}
	if err := db.Model(&entity.UserTenant{}).Where("id = ?", "user-tenant-1").Update("llm_id", defaultLLMID).Error; err != nil {
		t.Fatalf("failed to configure owner default model: %v", err)
	}
}

// configTools reports the is_tools flag getLLMModelConfig carries, read the way
// a consumer of that config map does.
func configTools(cfg map[string]interface{}) bool {
	v, _ := cfg["is_tools"].(bool)
	return v
}

// TestGetLLMModelConfigCarriesToolSupport pins that the resolution reports the
// capability of the row it actually loaded — the config the pipeline then carries
// through the request, which is where Python reads chat_mdl.is_tools from too.
func TestGetLLMModelConfigCarriesToolSupport(t *testing.T) {
	setupChatPipelineToolSupportTestDB(t)
	svc := NewChatPipelineService()

	for _, tc := range []struct {
		name      string
		llm       string
		wantTools bool
		wantType  string
	}{
		{"tool-capable model", "model-tools-on", true, "chat"},
		{"model without tool support", "model-tools-off", false, "chat"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			cfg, _, _, _, err := svc.getLLMModelConfig(t.Context(), &entity.Chat{TenantID: "tenant-1", LLMID: tc.llm})
			if err != nil {
				t.Fatalf("getLLMModelConfig(%s) failed: %v", tc.llm, err)
			}
			if got := configTools(cfg); got != tc.wantTools {
				t.Errorf("is_tools for %s = %v, want %v", tc.llm, got, tc.wantTools)
			}
			if got, _ := cfg["model_type"].(string); got != tc.wantType {
				t.Errorf("model_type for %s = %q, want %q", tc.llm, got, tc.wantType)
			}
		})
	}
}

// TestResolveModelConfigRejectsImage2TextOnlyModel ensures that a model
// enrolled only as image-to-text cannot be used as a chat model.
func TestResolveModelConfigRejectsImage2TextOnlyModel(t *testing.T) {
	setupChatPipelineToolSupportTestDB(t)
	svc := NewChatPipelineService()

	_, err := svc.ModelFactory.resolveConfig(
		permission.WithTenantID(t.Context(), "tenant-1"), entity.ModelTypeChat, "model-image2text-tools",
	)
	if err == nil {
		t.Fatal("ResolveModelConfig accepted an image2text-only model as chat")
	}
}

// TestGetLLMModelConfigRejectsUnusableModels pins the fail-closed rule: a disabled
// model, a model not enrolled as a chat model, and an unenrolled reference all
// fail to resolve (or resolve without a capability) instead of reporting tool
// support — the same outcome as Python's `getattr(chat_mdl, "is_tools", False)`
// default.
func TestGetLLMModelConfigRejectsUnusableModels(t *testing.T) {
	setupChatPipelineToolSupportTestDB(t)
	svc := NewChatPipelineService()

	for _, tc := range []struct {
		name string
		llm  string
	}{
		{"disabled model", "model-tools-disabled"},
		{"not enrolled as chat", "model-embedding-tools"},
		{"unknown model", "no-such-model"},
		{"unenrolled composite reference", "ghost@default@ZHIPU-AI"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			cfg, _, _, _, err := svc.getLLMModelConfig(t.Context(), &entity.Chat{TenantID: "tenant-1", LLMID: tc.llm})
			if err == nil && configTools(cfg) {
				t.Errorf("unusable model %s reported tool support", tc.llm)
			}
		})
	}
}

// TestGetLLMModelConfigFallsBackToTenantDefault covers the no-explicit-LLM branch:
// the resolution must fall back to the tenant default chat model (composite
// "model@provider" reference) and report its capability.
func TestGetLLMModelConfigFallsBackToTenantDefault(t *testing.T) {
	db := setupChatPipelineToolSupportTestDB(t)
	seedChatPipelineToolSupportTenant(t, db, "glm-4-plus@ZHIPU-AI")
	svc := NewChatPipelineService()

	cfg, _, _, _, err := svc.getLLMModelConfig(t.Context(), &entity.Chat{TenantID: "tenant-1"})
	if err != nil {
		t.Fatalf("getLLMModelConfig(tenant default) failed: %v", err)
	}
	if !configTools(cfg) {
		t.Error("tenant default with is_tools=true reported no tool support, want true")
	}
}

// TestGetLLMModelConfigTenantDefaultWithoutToolSupport is the mirror case: a
// tenant default that cannot call tools must not be reported as capable.
func TestGetLLMModelConfigTenantDefaultWithoutToolSupport(t *testing.T) {
	db := setupChatPipelineToolSupportTestDB(t)
	seedChatPipelineToolSupportTenant(t, db, "glm-4-flash@ZHIPU-AI")
	svc := NewChatPipelineService()

	cfg, _, _, _, err := svc.getLLMModelConfig(t.Context(), &entity.Chat{TenantID: "tenant-1"})
	if err != nil {
		t.Fatalf("getLLMModelConfig(tenant default) failed: %v", err)
	}
	if configTools(cfg) {
		t.Error("tenant default with is_tools=false reported tool support, want false")
	}
}

// TestResolveReasoningLevelReadsAgenticLevel pins where the agentic RAG mode is
// selected from. The level arrives as a number (frontend sends
// Number(getThinkingLevel())), from the request first and the dialog's
// prompt_config second, and reasoningLevelAgentic is the one level that means
// anything beyond "retrieve normally".
func TestResolveReasoningLevelReadsAgenticLevel(t *testing.T) {
	for _, tc := range []struct {
		name         string
		kwargs       map[string]interface{}
		promptConfig map[string]interface{}
		want         int
	}{
		{"absent everywhere", nil, nil, 0},
		{"naive from request", map[string]interface{}{"reasoning": 0}, nil, 0},
		{"stale level from request", map[string]interface{}{"reasoning": 3}, nil, 3},
		{"agentic from request", map[string]interface{}{"reasoning": 5}, nil, reasoningLevelAgentic},
		{
			"request float from JSON",
			map[string]interface{}{"reasoning": float64(reasoningLevelAgentic)},
			nil,
			reasoningLevelAgentic,
		},
		{
			"agentic from dialog config",
			nil,
			map[string]interface{}{"reasoning": 5},
			reasoningLevelAgentic,
		},
		{
			// The UI always sends the level, so a request value must win over
			// a dialog pinned to a different mode.
			"request overrides dialog config",
			map[string]interface{}{"reasoning": 2},
			map[string]interface{}{"reasoning": 5},
			2,
		},
		// chat_settings.ts writes a boolean, which must not read as agentic.
		{"boolean false is not agentic", nil, map[string]interface{}{"reasoning": false}, 0},
	} {
		t.Run(tc.name, func(t *testing.T) {
			got := resolveReasoningLevel(tc.kwargs, tc.promptConfig)
			if got != tc.want {
				t.Errorf("resolveReasoningLevel() = %d, want %d", got, tc.want)
			}
		})
	}
}

// TestQuoteEnabledNeedsBothSourcesToAgree pins the effective citation setting:
// the request and the dialog each get a veto, and an absent setting on either
// side leaves citations on.
//
// The agentic dispatch resolves this before the retrieval phases, so it is the
// same function Phase 6 uses. When the agentic branch resolved it separately
// (or not at all), a request's quote:false was ignored and every agentic answer
// still shipped [ID:N] markers plus a reference payload.
func TestQuoteEnabledNeedsBothSourcesToAgree(t *testing.T) {
	for _, tc := range []struct {
		name         string
		kwargs       map[string]interface{}
		promptConfig map[string]interface{}
		want         bool
	}{
		{"neither source", nil, nil, true},
		{"request on", map[string]interface{}{"quote": true}, nil, true},
		{"request off", map[string]interface{}{"quote": false}, nil, false},
		{"dialog off", nil, map[string]interface{}{"quote": false}, false},
		{"request off wins over dialog on", map[string]interface{}{"quote": false}, map[string]interface{}{"quote": true}, false},
		// The dialog is a second veto, not a way to re-enable what the request
		// turned off.
		{"dialog off wins over request on", map[string]interface{}{"quote": true}, map[string]interface{}{"quote": false}, false},
		// A non-bool is not a decision; ignore it rather than disabling citations.
		{"non-bool request value", map[string]interface{}{"quote": "false"}, nil, true},
	} {
		t.Run(tc.name, func(t *testing.T) {
			if got := quoteEnabled(tc.kwargs, tc.promptConfig); got != tc.want {
				t.Errorf("quoteEnabled() = %v, want %v", got, tc.want)
			}
		})
	}
}

// TestAgenticDispatchRequiresKnowledgeBaseScope pins that the agentic branch is
// unreachable without a knowledge base.
//
// The agent resolves a citation against an explicit kb_id scope, and
// buildBoolQueryFromCondition drops that term when the scope is empty — so an
// agentic turn on a KB-less dialog would resolve a cited chunk out of ANY KB in
// the tenant. The dispatch therefore falls through instead of running the agent
// with an unbounded scope, which keeps the empty-scope query unconstructible
// rather than trusting each retrieval path to defend itself.
func TestAgenticDispatchRequiresKnowledgeBaseScope(t *testing.T) {
	for _, tc := range []struct {
		name   string
		hasKBs bool
		kwargs map[string]interface{}
		want   bool
	}{
		{"agentic level with KBs", true, map[string]interface{}{"reasoning": reasoningLevelAgentic}, true},
		{"agentic level without KBs", false, map[string]interface{}{"reasoning": reasoningLevelAgentic}, false},
		{"explicit agent_mode without KBs", false, map[string]interface{}{"agent_mode": "smart-reasoning"}, false},
		{"explicit agent_mode with KBs", true, map[string]interface{}{"agent_mode": "smart-reasoning"}, true},
		{"no selection without KBs", false, map[string]interface{}{"reasoning": 0}, false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			// Mirrors the guard in AsyncChat's agentic branch.
			_, byMode := tc.kwargs["agent_mode"].(string)
			byLevel := resolveReasoningLevel(tc.kwargs, nil) == reasoningLevelAgentic
			dispatches := (byMode || byLevel) && tc.hasKBs
			if dispatches != tc.want {
				t.Errorf("dispatches to agentic = %v, want %v (byMode=%v byLevel=%v hasKBs=%v)",
					dispatches, tc.want, byMode, byLevel, tc.hasKBs)
			}
		})
	}
}

// TestGetLLMModelConfigReadsToolSupportPerResolution pins the consequence of
// carrying the flag on the resolution instead of memoizing it: a model edited
// after the first request is seen by the next resolution, with no TTL to wait out.
func TestGetLLMModelConfigReadsToolSupportPerResolution(t *testing.T) {
	db := setupChatPipelineToolSupportTestDB(t)
	svc := NewChatPipelineService()
	chat := &entity.Chat{TenantID: "tenant-1", LLMID: "model-tools-on"}

	cfg, _, _, _, err := svc.getLLMModelConfig(t.Context(), chat)
	if err != nil {
		t.Fatalf("getLLMModelConfig failed: %v", err)
	}
	if !configTools(cfg) {
		t.Fatal("tool-capable model reported no support")
	}

	if err := db.Model(&entity.TenantModel{}).Where("id = ?", "model-tools-on").
		Update("extra", `{"is_tools":false}`).Error; err != nil {
		t.Fatalf("flip is_tools: %v", err)
	}

	cfg, _, _, _, err = svc.getLLMModelConfig(t.Context(), chat)
	if err != nil {
		t.Fatalf("getLLMModelConfig after the flag flip failed: %v", err)
	}
	if configTools(cfg) {
		t.Error("a flipped is_tools flag was not seen by the next resolution")
	}
}

// ttsSpyDriver records the text a TTS driver is asked to synthesize.
type ttsSpyDriver struct {
	modelModule.ModelDriver
	input  string
	called bool
}

func (d *ttsSpyDriver) AudioSpeech(_ context.Context, _ *string, audioContent *string, _ *modelModule.APIConfig, _ *modelModule.TTSConfig, _ *common.ModelUsage) (*modelModule.TTSResponse, error) {
	d.input = *audioContent
	d.called = true
	return &modelModule.TTSResponse{Audio: []byte("audio")}, nil
}

// TestSynthesizeTTS_CutsAtFiveHundredCharacters pins the cap to 500 characters,
// so a CJK answer keeps its length and never ends in half a rune.
func TestSynthesizeTTS_CutsAtFiveHundredCharacters(t *testing.T) {
	spy := &ttsSpyDriver{}
	name := "tts"
	ttsModel := modelModule.NewTTSModel(spy, &name, nil)
	s := &ChatPipelineService{}
	if audio := s.synthesizeTTS(t.Context(), ttsModel, strings.Repeat("知", 600)); audio == nil {
		t.Fatal("expected audio")
	}
	if want := strings.Repeat("知", 500); spy.input != want {
		t.Errorf("TTS input is %d bytes / %d runes, want 500 whole characters", len(spy.input), len([]rune(spy.input)))
	}
}

func TestCleanTTSText(t *testing.T) {
	for _, tc := range []struct {
		name string
		text string
		want string
	}{
		{name: "empty"},
		{name: "ASCII below limit", text: strings.Repeat("a", 499), want: strings.Repeat("a", 499)},
		{name: "ASCII at limit", text: strings.Repeat("a", 500), want: strings.Repeat("a", 500)},
		{name: "ASCII over limit", text: strings.Repeat("a", 501), want: strings.Repeat("a", 500)},
		{name: "CJK below limit", text: strings.Repeat("知", 499), want: strings.Repeat("知", 499)},
		{name: "CJK at limit", text: strings.Repeat("知", 500), want: strings.Repeat("知", 500)},
		{name: "CJK over limit", text: strings.Repeat("知", 501), want: strings.Repeat("知", 500)},
		{name: "mixed rune widths", text: strings.Repeat("aé知𐐀", 126), want: strings.Repeat("aé知𐐀", 125)},
		{name: "control emoji and whitespace", text: "\x00\t你好🙂\nworld\x7f  ", want: "你好 world"},
		{name: "tags", text: "<think>reason</think> <b>answer</b>", want: "reason answer"},
		{name: "comparisons", text: "2 < 3 and 4 > 1; a < b and c > d", want: "2 < 3 and 4 > 1; a < b and c > d"},
		{name: "compact comparisons", text: "a<b>c; x<value>y", want: "a<b>c; x<value>y"},
		{name: "unclosed tag", text: "<B>answer", want: "<B>answer"},
		{name: "unmatched closing tag", text: "a</b>c", want: "a</b>c"},
		{name: "nested tags", text: "<B><i>answer</i></b>", want: "answer"},
		{name: "unmatched outer tag", text: "<b>unclosed <b>paired</b>", want: "<b>unclosed paired"},
		{name: "encoded comparisons", text: "a&lt;b&gt;c", want: "a&lt;b&gt;c"},
		{name: "incomplete tag", text: `answer <b title="unfinished`, want: `answer <b title="unfinished`},
		{name: "quoted attributes", text: `<b title="a > b">answer</b> <span title='c > d'>text</span><br/>`, want: "answer text"},
		{name: "trailing reasoning tag", text: "答案</think>", want: "答案"},
		{name: "invalid UTF-8", text: "\xff知\xc3é\xe7\x9f", want: "知é"},
		{name: "valid replacement character", text: "知\ufffdé", want: "知\ufffdé"},
		{name: "cleanup before limit", text: "<think>" + strings.Repeat("知", 499) + "</think>\xff界🙂\x00余", want: strings.Repeat("知", 499) + "界"},
		{name: "cleanup to empty", text: "<think></think>🙂\xff\x00 \t\n"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			if got := cleanTTSText(tc.text); got != tc.want {
				t.Errorf("cleanTTSText() = %q, want %q", got, tc.want)
			}
		})
	}
}

func TestSynthesizeTTS_CleansTextBeforeCallingDriver(t *testing.T) {
	for _, tc := range []struct {
		name string
		text string
		want string
	}{
		{name: "mixed answer", text: "\xff<think>思考</think> \x00hello🙂", want: "思考 hello"},
		{name: "compact comparison", text: "a<b>c", want: "a<b>c"},
		{name: "empty"},
		{name: "cleanup to empty", text: "<think></think>🙂\xff\x00 \t\n"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			spy := &ttsSpyDriver{}
			name := "tts"
			ttsModel := modelModule.NewTTSModel(spy, &name, nil)
			s := &ChatPipelineService{}
			audio := s.synthesizeTTS(t.Context(), ttsModel, tc.text)
			wantCall := tc.want != ""
			if spy.called != wantCall {
				t.Errorf("TTS driver called = %v, want %v", spy.called, wantCall)
			}
			if spy.input != tc.want {
				t.Errorf("TTS input = %q, want %q", spy.input, tc.want)
			}
			if (audio != nil) != wantCall {
				t.Errorf("audio = %v, want audio present = %v", audio, wantCall)
			}
		})
	}
}

func TestAsyncChat_ContextCancellationTerminatesGoroutine(t *testing.T) {
	db := setupChatPipelineVisionTestDB(t)
	seedChatPipelineVisionTenant(t, db, "glm-4-flash@ZHIPU-AI")

	ctx, cancel := context.WithCancel(context.Background())
	cancel() // pre-cancel context to simulate client disconnect

	chat := dialForTest("glm-4-flash@ZHIPU-AI")
	results, err := NewChatPipelineService().AsyncChat(ctx, chat,
		[]map[string]interface{}{{"role": "user", "content": "Question"}}, true, nil)
	if err != nil {
		t.Fatal(err)
	}

	// Do NOT drain results with a background goroutine.
	// AsyncChat must detect pre-cancelled context, exit, and close results.
	timeout := time.After(2 * time.Second)
	for {
		select {
		case _, ok := <-results:
			if !ok {
				// success: AsyncChat goroutine exited cleanly without leaking
				return
			}
		case <-timeout:
			t.Fatal("AsyncChat goroutine leaked: did not terminate when context was cancelled")
		}
	}
}