Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

36.5 KB
Raw Blame
//
//  Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.
//

// Tests for the message-building / flag-normalisation helpers used
// by BotService.ChatbotCompletion. A resumed session_id must carry
// prior turns (assistant prologue + earlier user/assistant
// exchanges) into the next pipeline call so multi-turn chatbot
// clients retain context, and the filtering must match python
// async_iframe_completion (drop system turns and the leading
// assistant prologue).

package service

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net/http"
	"reflect"
	"strings"
	"sync"
	"testing"
	"time"

	"ragflow/internal/agent/canvas"
	"ragflow/internal/common"
	"ragflow/internal/dao"
	"ragflow/internal/entity"
)

func TestBuildChatbotPipelineMessages_Empty(t *testing.T) {
	// A freshly-seeded session with no prior turns produces just
	// the new user question. Matches python conversation_service.
	msgs := buildChatbotPipelineMessages(nil, "hi", "msg-1")
	if len(msgs) != 1 {
		t.Fatalf("nil raw: want 1 message, got %d", len(msgs))
	}
	if msgs[0]["role"] != "user" || msgs[0]["content"] != "hi" || msgs[0]["id"] != "msg-1" {
		t.Errorf("got %+v", msgs[0])
	}

	msgs = buildChatbotPipelineMessages(json.RawMessage(`[]`), "hi", "msg-1")
	if len(msgs) != 1 {
		t.Fatalf("empty array: want 1 message, got %d", len(msgs))
	}
}

func TestBuildChatbotPipelineMessages_DropsLeadingAssistantAndSystem(t *testing.T) {
	// Prologue (leading assistant) and system turns must not reach
	// the pipeline; later assistant turns are kept so the LLM sees
	// prior exchanges.
	turns := []map[string]any{
		{"role": "assistant", "content": "Hello, how can I help?", "created_at": 1},
		{"role": "system", "content": "hidden", "created_at": 2},
		{"role": "user", "content": "What is Go?", "created_at": 3},
		{"role": "assistant", "content": "Go is a compiled language.", "created_at": 4, "reference": map[string]any{"chunks": []any{"large chunk"}}, "prompt": "system", "usage": map[string]any{"total_tokens": 10}, "elapsed_ms": 100},
	}
	raw, err := json.Marshal(turns)
	if err != nil {
		t.Fatalf("marshal seed: %v", err)
	}
	msgs := buildChatbotPipelineMessages(raw, "and Rust?", "msg-2")
	if len(msgs) != 3 {
		t.Fatalf("want 3 messages (user, assistant, new user), got %d: %+v", len(msgs), msgs)
	}
	if msgs[0]["role"] != "user" || msgs[0]["content"] != "What is Go?" {
		t.Errorf("turn 0: %+v", msgs[0])
	}
	if msgs[1]["role"] != "assistant" || msgs[1]["content"] != "Go is a compiled language." {
		t.Errorf("turn 1: %+v", msgs[1])
	}
	for _, key := range []string{"prompt", "usage", "elapsed_ms", "reference"} {
		if _, ok := msgs[1][key]; ok {
			t.Fatalf("LLM history contains %s: %v", key, msgs[1])
		}
	}
	if msgs[2]["role"] != "user" || msgs[2]["content"] != "and Rust?" || msgs[2]["id"] != "msg-2" {
		t.Errorf("turn 2: %+v", msgs[2])
	}
}

func TestBuildChatbotPipelineMessages_Malformed(t *testing.T) {
	// Malformed JSON must not panic; falls back to just the new
	// question rather than failing the request.
	msgs := buildChatbotPipelineMessages(json.RawMessage(`not json`), "hi", "msg-3")
	if len(msgs) != 1 || msgs[0]["content"] != "hi" {
		t.Fatalf("malformed raw: want single user turn, got %+v", msgs)
	}
}

func TestNormalizeBotBoolFlag(t *testing.T) {
	cases := []struct {
		in    any
		value bool
		ok    bool
	}{
		{true, true, true},
		{false, false, true},
		{float64(1), true, true},
		{float64(0), false, true},
		{1, true, true},
		{0, false, true},
		{nil, false, false},
		{"yes", false, false},
		{float64(2), false, false},
	}
	for _, c := range cases {
		value, ok := normalizeBotBoolFlag(c.in)
		if value != c.value || ok != c.ok {
			t.Errorf("normalizeBotBoolFlag(%v) = (%v, %v), want (%v, %v)",
				c.in, value, ok, c.value, c.ok)
		}
	}
}

// TestBotService_AgentbotInputs_CrossTenantDenied mirrors PR
// #15457: when a beta API token authenticates a caller with
// tenantID, that caller must not be able to read an agent
// belonging to a different tenant. The Go guard runs inside
// loadCanvas (called at the entry of AgentbotInputs and
// AgentbotCompletion) and returns ErrUserCanvasNotFound — same
// 404-equivalent shape as the python fix returns "Can't find
// agent by ID: <id>". This test seeds a canvas under tenant-A
// and asks for it via tenant-B; the call must fail with the
// not-found error and never expose the canvas.
func TestBotService_AgentbotInputs_CrossTenantDenied(t *testing.T) {
	db := setupServiceTestDB(t)
	if err := db.AutoMigrate(
		&entity.UserCanvas{},
		&entity.UserTenant{},
	); err != nil {
		t.Fatalf("migrate: %v", err)
	}
	orig := dao.DB
	dao.DB = db
	t.Cleanup(func() { dao.DB = orig })

	// Seed tenant-A and a canvas owned by user-A.
	if err := db.Create(&entity.UserTenant{
		ID:       "ut-A",
		UserID:   "user-A",
		TenantID: "tenant-A",
		Role:     "owner",
	}).Error; err != nil {
		t.Fatalf("seed tenant-A: %v", err)
	}
	if err := db.Create(&entity.UserCanvas{
		ID:             "agent-victim",
		UserID:         "user-A",
		Title:          sptr("Victim Agent"),
		CanvasCategory: "agent_canvas",
	}).Error; err != nil {
		t.Fatalf("seed victim agent: %v", err)
	}

	// Seed tenant-B (the attacker's tenant).
	if err := db.Create(&entity.UserTenant{
		ID:       "ut-B",
		UserID:   "user-B",
		TenantID: "tenant-B",
		Role:     "owner",
	}).Error; err != nil {
		t.Fatalf("seed tenant-B: %v", err)
	}

	svc := NewBotService(nil)

	// Attacker (tenant-B) asks for victim (tenant-A's canvas).
	title, _, _, _, _, code, err := svc.AgentbotInputs(serviceTestContext(t, "tenant-B", "tenant-B"),
		"agent-victim")
	if !errors.Is(err, dao.ErrUserCanvasNotFound) {
		t.Errorf("cross-tenant: want ErrUserCanvasNotFound, got %v", err)
	}
	if code != common.CodeDataError {
		t.Errorf("cross-tenant: want code %d, got %d", common.CodeDataError, code)
	}
	if title != "" {
		t.Errorf("cross-tenant: title should be empty, got %q (data leak)", title)
	}
}

func TestBotService_AgentbotInputsReadsBeginParams(t *testing.T) {
	db := setupServiceTestDB(t)
	if err := db.AutoMigrate(
		&entity.UserCanvas{},
		&entity.UserTenant{},
	); err != nil {
		t.Fatalf("migrate: %v", err)
	}
	orig := dao.DB
	dao.DB = db
	t.Cleanup(func() { dao.DB = orig })

	if err := db.Create(&entity.UserTenant{
		ID:       "ut-owner",
		UserID:   "owner-1",
		TenantID: "tenant-1",
		Role:     "owner",
	}).Error; err != nil {
		t.Fatalf("seed tenant: %v", err)
	}

	title := "Agent"
	avatar := "avatar.png"
	if err := db.Create(&entity.UserCanvas{
		ID:             "agent-1",
		UserID:         "owner-1",
		TenantID:       sptr("tenant-1"),
		Title:          &title,
		Avatar:         &avatar,
		CanvasCategory: "agent_canvas",
		DSL: entity.JSONMap{
			"components": map[string]any{
				"begin": map[string]any{
					"obj": map[string]any{
						"component_name": "Begin",
						"params": map[string]any{
							"mode":     "conversational",
							"prologue": "Welcome from the agent",
							"inputs": map[string]any{
								"topic": map[string]any{"type": "line", "name": "Topic"},
							},
						},
					},
				},
			},
		},
	}).Error; err != nil {
		t.Fatalf("seed agent: %v", err)
	}

	svc := NewBotService(nil)
	gotTitle, gotAvatar, prologue, mode, inputs, code, err := svc.AgentbotInputs(
		serviceTestContext(t, "owner-1", "tenant-1"), "agent-1")
	if err != nil {
		t.Fatalf("AgentbotInputs: %v", err)
	}
	if code != common.CodeSuccess {
		t.Fatalf("code = %d, want %d", code, common.CodeSuccess)
	}
	if gotTitle != title || gotAvatar != avatar {
		t.Fatalf("title/avatar = %q/%q, want %q/%q", gotTitle, gotAvatar, title, avatar)
	}
	if prologue != "Welcome from the agent" {
		t.Errorf("prologue = %q, want Welcome from the agent", prologue)
	}
	if mode != "conversational" {
		t.Errorf("mode = %q, want conversational", mode)
	}
	if _, ok := inputs["topic"].(map[string]any); !ok {
		t.Errorf("inputs = %#v, want topic input", inputs)
	}
}

// TestWriteChatbotRunEvent_UserInputsEvent guards PR #14589: the
// SSE envelope must carry the canvas event type so the front-end
// can distinguish interactive "user_inputs" / "workflow_finished"
// events (which need a UserFillUp form) from plain "message"
// events (assistant text). Without the `event` field the form
// UI never appears and the canvas appears to hang.
func TestWriteChatbotRunEvent_UserInputsEvent(t *testing.T) {
	rec := &recordingResponseWriter{header: http.Header{}}
	if err := WriteChatbotRunEvent(rec, canvas.RunEvent{
		Type:      "user_inputs",
		MessageID: "msg-1",
		Data:      `{"components":[{"id":"email","type":"text","required":true}]}`,
		SessionID: "sess-1",
	}); err != nil {
		t.Fatalf("WriteChatbotRunEvent: %v", err)
	}
	body := rec.body.String()
	if !strings.Contains(body, `"event":"user_inputs"`) {
		t.Errorf("body missing event=user_inputs: %s", body)
	}
	if !strings.Contains(body, `"message_id":"msg-1"`) {
		t.Errorf("body missing message_id: %s", body)
	}
	if !strings.Contains(body, `"task_id":"sess-1"`) {
		t.Errorf("body missing session-backed task_id alias: %s", body)
	}
	if !strings.Contains(body, `"session_id":"sess-1"`) {
		t.Errorf("body missing session_id: %s", body)
	}
	if strings.Contains(body, `"answer":"`) {
		t.Errorf("body should not wrap run events in data.answer: %s", body)
	}
}

// TestWriteChatbotRunEvent_WorkflowFinishedEvent covers the second
// new event type from PR #14589. The envelope must also carry
// "workflow_finished" verbatim.
func TestWriteChatbotRunEvent_WorkflowFinishedEvent(t *testing.T) {
	rec := &recordingResponseWriter{header: http.Header{}}
	if err := WriteChatbotRunEvent(rec, canvas.RunEvent{
		Type:      "workflow_finished",
		Data:      `{"outputs":"done"}`,
		SessionID: "sess-2",
	}); err != nil {
		t.Fatalf("WriteChatbotRunEvent: %v", err)
	}
	body := rec.body.String()
	if !strings.Contains(body, `"event":"workflow_finished"`) {
		t.Errorf("body missing event=workflow_finished: %s", body)
	}
	if !strings.Contains(body, `"outputs":"done"`) {
		t.Errorf("body missing workflow output payload: %s", body)
	}
}

// TestWriteChatbotRunEvent_MessageEventCarriesEvent ensures the
// existing "message" path also carries the event field. The
// front-end can rely on `data.event` to distinguish message
// frames from user_inputs / workflow_finished frames without
// a separate header.
func TestWriteChatbotRunEvent_MessageEventCarriesEvent(t *testing.T) {
	rec := &recordingResponseWriter{header: http.Header{}}
	if err := WriteChatbotRunEvent(rec, canvas.RunEvent{
		Type:      "message",
		MessageID: "msg-3",
		Data:      `{"content":"hi"}`,
		SessionID: "sess-3",
	}); err != nil {
		t.Fatalf("WriteChatbotRunEvent: %v", err)
	}
	body := rec.body.String()
	if !strings.Contains(body, `"event":"message"`) {
		t.Errorf("message frame should carry event=message: %s", body)
	}
	if !strings.Contains(body, `"message_id":"msg-3"`) {
		t.Errorf("message frame should carry top-level message_id: %s", body)
	}
	if !strings.Contains(body, `"content":"hi"`) {
		t.Errorf("message frame should carry data.content: %s", body)
	}
}

func TestWriteChatbotRunEvent_ErrorCarriesSessionAlias(t *testing.T) {
	rec := &recordingResponseWriter{header: http.Header{}}
	if err := WriteChatbotRunEvent(rec, canvas.RunEvent{
		Type:      "error",
		Data:      `{"message":"failed"}`,
		SessionID: "sess-error",
	}); err != nil {
		t.Fatalf("WriteChatbotRunEvent: %v", err)
	}
	body := rec.body.String()
	if !strings.Contains(body, `"task_id":"sess-error"`) ||
		!strings.Contains(body, `"session_id":"sess-error"`) {
		t.Fatalf("error frame missing session-backed aliases: %s", body)
	}
}

func TestWriteChatbotRunEvent_RedactsInternalError(t *testing.T) {
	rec := &recordingResponseWriter{header: http.Header{}}
	if err := WriteChatbotRunEvent(rec, canvas.RunEvent{
		Type: "error",
		Data: `{"message":"dial mysql.internal:3306 password=secret","kind":"internal"}`,
	}); err != nil {
		t.Fatalf("WriteChatbotRunEvent: %v", err)
	}
	body := rec.body.String()
	if !strings.Contains(body, canvas.InternalRunErrorMessage) {
		t.Errorf("body missing safe internal error message: %s", body)
	}
	if strings.Contains(body, "mysql.internal") || strings.Contains(body, "password=secret") {
		t.Fatalf("body leaked internal error details: %s", body)
	}
}

// TestBotService_ChatbotCompletion_NewSessionSkipsLLM locks in the
// share-page handshake behaviour: the front-end opens a shared chat
// with an empty question and no session_id only to obtain a session
// (web/src/pages/next-chats/hooks/use-send-shared-message.ts
// fetchSessionId). Mirrors python async_iframe_completion
// (conversation_service.py:324-334): the server must create the
// prologue-seeded session and stream the prologue back WITHOUT
// invoking the LLM. The previous Go implementation ran the model
// with an empty user turn, so merely opening a share link produced
// a fabricated "you sent a blank message" reply.
func TestBotService_ChatbotCompletion_NewSessionSkipsLLM(t *testing.T) {
	db := setupServiceTestDB(t)
	if err := db.AutoMigrate(&entity.Chat{}); err != nil {
		t.Fatalf("migrate chat: %v", err)
	}
	pushServiceDB(t, db)

	prologue := "你好!我是你的AI助手。"
	if err := db.Create(&entity.Chat{
		ID:           "dlg-1",
		TenantID:     "tenant-1",
		Name:         sptr("bot"),
		LLMSetting:   entity.JSONMap{},
		PromptType:   "simple",
		PromptConfig: entity.JSONMap{"prologue": prologue},
		KBIDs:        entity.JSONSlice{},
		Status:       sptr(common.StatusDialogValid),
	}).Error; err != nil {
		t.Fatalf("seed dialog: %v", err)
	}

	// llmService is nil — any attempt to reach the LLM path would
	// fail loudly, so a successful run proves the LLM was skipped.
	svc := NewBotService(nil)
	frames, code, err := svc.ChatbotCompletion(serviceTestContext(t, "tenant-1", "tenant-1"),
		"dlg-1", ChatbotCompletionRequest{Question: ""})
	if err != nil {
		t.Fatalf("ChatbotCompletion: %v", err)
	}
	if code != common.CodeSuccess {
		t.Fatalf("want code %d, got %d", common.CodeSuccess, code)
	}

	var got []ChatbotSSEFrame
	for f := range frames {
		got = append(got, f)
	}
	if len(got) != 2 {
		t.Fatalf("want 2 frames (prologue + done), got %d: %+v", len(got), got)
	}
	if got[0].Err != nil {
		t.Errorf("frame 0 unexpected error: %v", got[0].Err)
	}
	if got[0].Data != prologue {
		t.Errorf("frame 0 data = %q, want prologue %q", got[0].Data, prologue)
	}
	if got[0].SessionID == "" {
		t.Errorf("frame 0 must carry the new session_id")
	}
	if !got[1].Done {
		t.Errorf("frame 1 must be the done marker")
	}

	// The persisted session must hold exactly the prologue turn —
	// no empty user message may be written.
	ctx := t.Context()
	row, derr := dao.NewAPI4ConversationDAO().GetBySessionID(ctx, dao.DB, got[0].SessionID, "dlg-1", true)
	if derr != nil || row == nil {
		t.Fatalf("session not persisted: row=%v err=%v", row, derr)
	}
	msgs := parseChatbotTurns(row.Message)
	if len(msgs) != 1 || msgs[0]["role"] != "assistant" || msgs[0]["content"] != prologue {
		t.Errorf("persisted messages = %+v, want single prologue assistant turn", msgs)
	}
}

func TestParseChatbotTurns(t *testing.T) {
	// Empty / malformed input must yield an empty (non-nil) slice so
	// callers can always append.
	for _, raw := range []json.RawMessage{
		nil,
		json.RawMessage(`[]`),
		json.RawMessage(`null`),
		json.RawMessage(`not json`),
	} {
		turns := parseChatbotTurns(raw)
		if turns == nil {
			t.Errorf("parseChatbotTurns(%q) returned a nil slice", string(raw))
		}
		if len(turns) != 0 {
			t.Errorf("parseChatbotTurns(%q) = %+v, want empty", string(raw), turns)
		}
	}

	turns := parseChatbotTurns(json.RawMessage(`[{"role":"user","content":"hi"}]`))
	if len(turns) != 1 || turns[0]["role"] != "user" || turns[0]["content"] != "hi" {
		t.Errorf("valid input: got %+v", turns)
	}
}

func TestReferenceOrEmpty(t *testing.T) {
	got := referenceOrEmpty(nil)
	if got == nil || len(got) != 0 {
		t.Errorf("nil reference must yield an empty non-nil map, got %v", got)
	}
	ref := map[string]any{"chunks": []any{map[string]any{"chunk_id": "c1"}}}
	got = referenceOrEmpty(ref)
	chunks, _ := got["chunks"].([]any)
	if len(chunks) != 1 {
		t.Errorf("non-nil reference must pass through, got %+v", got)
	}
}

// TestPersistChatbotTurn_AppendsPairAndReference covers the turn
// persistence after a completed stream: the user/assistant pair is
// appended to the existing history, shares one message id, and the
// retrieval reference is appended to the reference list.
func TestPersistChatbotTurn_AppendsPairAndReference(t *testing.T) {
	db := setupServiceTestDB(t)
	pushServiceDB(t, db)

	seed, _ := json.Marshal([]map[string]any{
		{"role": "assistant", "content": "Hello!", "created_at": 1},
	})
	sess := &entity.API4Conversation{
		ID:       "sess-p1",
		DialogID: "dlg-p1",
		UserID:   "tenant-1",
		Message:  seed,
	}
	ctx := t.Context()
	if err := dao.NewAPI4ConversationDAO().Create(ctx, dao.DB, sess); err != nil {
		t.Fatalf("seed session: %v", err)
	}

	svc := NewBotService(nil)
	ref := map[string]any{
		"chunks":   []any{map[string]any{"chunk_id": "c1"}},
		"doc_aggs": []any{},
	}
	if err := svc.persistChatbotQuestion(ctx, sess, "What is Go?", "msg-p1", 2); err != nil {
		t.Fatalf("persist question: %v", err)
	}
	questionRow, err := dao.NewAPI4ConversationDAO().GetBySessionID(ctx, dao.DB, "sess-p1", "dlg-p1", true, dao.ConversationHistoryOptions{IncludeReference: true})
	if err != nil || questionRow == nil {
		t.Fatalf("read pending question: %v", err)
	}
	if turns := parseChatbotTurns(questionRow.Message); len(turns) != 2 || turns[1]["role"] != "user" || turns[1]["created_at"] != float64(2) {
		t.Fatalf("question must be persisted before completion: %+v", turns)
	}
	if err := svc.persistChatbotTurn(ctx, sess, "What is Go?", "A language.", "msg-p1", ref, 3); err != nil {
		t.Fatalf("persistChatbotTurn: %v", err)
	}

	row, err := dao.NewAPI4ConversationDAO().GetBySessionID(ctx, dao.DB, "sess-p1", "dlg-p1", true, dao.ConversationHistoryOptions{IncludeReference: true})
	if err != nil || row == nil {
		t.Fatalf("re-read session: row=%v err=%v", row, err)
	}
	turns := parseChatbotTurns(row.Message)
	if len(turns) != 3 {
		t.Fatalf("want 3 turns (prologue + pair), got %d: %+v", len(turns), turns)
	}
	// The Q&A pair shares the message id so the exchange is addressed
	// as a unit, matching the in-app chat pairing convention.
	if turns[1]["role"] != "user" || turns[1]["content"] != "What is Go?" || turns[1]["id"] != "msg-p1" {
		t.Errorf("user turn: %+v", turns[1])
	}
	if turns[2]["role"] != "assistant" || turns[2]["content"] != "A language." || turns[2]["id"] != "msg-p1" {
		t.Errorf("assistant turn: %+v", turns[2])
	}
	if turns[1]["created_at"] != float64(2) || turns[2]["created_at"] != float64(3) {
		t.Fatalf("question and answer timestamps must remain separate: %+v", turns)
	}

	var refs []map[string]any
	for _, message := range parseChatbotTurns(row.Message) {
		if reference, ok := message["reference"].(map[string]any); ok {
			refs = append(refs, reference)
		}
	}
	if len(refs) != 1 {
		t.Fatalf("want 1 reference entry, got %d: %+v", len(refs), refs)
	}
	chunks, _ := refs[0]["chunks"].([]any)
	if len(chunks) != 1 {
		t.Errorf("reference chunks: %+v", refs[0])
	}
}

// TestPersistChatbotTurn_NilReferenceDefaultsToEmpty ensures a turn
// without retrieval results still records an empty reference object,
// attached directly to its assistant message.
func TestPersistChatbotTurn_NilReferenceDefaultsToEmpty(t *testing.T) {
	db := setupServiceTestDB(t)
	pushServiceDB(t, db)

	ctx := t.Context()
	sess := &entity.API4Conversation{ID: "sess-p2", DialogID: "dlg-p1", UserID: "tenant-1"}
	if err := dao.NewAPI4ConversationDAO().Create(ctx, dao.DB, sess); err != nil {
		t.Fatalf("seed session: %v", err)
	}

	svc := NewBotService(nil)
	if err := svc.persistChatbotQuestion(ctx, sess, "q", "msg-p2", 2); err != nil {
		t.Fatalf("persist question: %v", err)
	}
	if err := svc.persistChatbotTurn(ctx, sess, "q", "a", "msg-p2", nil, 3); err != nil {
		t.Fatalf("persistChatbotTurn: %v", err)
	}

	row, err := dao.NewAPI4ConversationDAO().GetBySessionID(ctx, dao.DB, "sess-p2", "dlg-p1", true, dao.ConversationHistoryOptions{IncludeReference: true})
	if err != nil || row == nil {
		t.Fatalf("re-read session: row=%v err=%v", row, err)
	}
	var refs []map[string]any
	for _, message := range parseChatbotTurns(row.Message) {
		if reference, ok := message["reference"].(map[string]any); ok {
			refs = append(refs, reference)
		}
	}
	if len(refs) != 1 {
		t.Fatalf("want 1 reference entry, got %+v", refs)
	}
	if chunks, ok := refs[0]["chunks"].([]any); !ok || len(chunks) != 0 {
		t.Errorf("default reference must carry empty chunks: %+v", refs[0])
	}
	if aggs, ok := refs[0]["doc_aggs"].([]any); !ok || len(aggs) != 0 {
		t.Errorf("default reference must carry empty doc_aggs: %+v", refs[0])
	}
}

// TestPersistChatbotTurn_ConcurrentSameSession guards the
// read-modify-write race on session.Message: two requests persisting
// to the same session_id at the same time must both land — without
// the persist lock + re-read the last Update would silently drop one
// exchange.
func TestPersistChatbotTurn_ConcurrentSameSession(t *testing.T) {
	db := setupServiceTestDB(t)
	pushServiceDB(t, db)
	// In-memory sqlite opens a fresh database per connection, so pin
	// the pool to a single connection for the concurrent goroutines.
	sqlDB, err := db.DB()
	if err != nil {
		t.Fatalf("db handle: %v", err)
	}
	sqlDB.SetMaxOpenConns(1)

	ctx := t.Context()
	sess := &entity.API4Conversation{ID: "sess-p3", DialogID: "dlg-p1", UserID: "tenant-1"}
	if err = dao.NewAPI4ConversationDAO().Create(ctx, dao.DB, sess); err != nil {
		t.Fatalf("seed session: %v", err)
	}

	svc := NewBotService(nil)
	// Both callers hold the same stale snapshot loaded before their
	// streams started — exactly the race the lock fixes.
	var wg sync.WaitGroup
	for i, turn := range [][2]string{{"q1", "a1"}, {"q2", "a2"}} {
		wg.Add(1)
		go func(i int, q, a string) {
			defer wg.Done()
			if err := svc.persistChatbotQuestion(ctx, sess, q, fmt.Sprintf("msg-p3-%d", i), 2); err != nil {
				t.Errorf("persist question %d: %v", i, err)
				return
			}
			if err := svc.persistChatbotTurn(ctx, sess, q, a, fmt.Sprintf("msg-p3-%d", i), nil, 3); err != nil {
				t.Errorf("persistChatbotTurn %d: %v", i, err)
			}
		}(i, turn[0], turn[1])
	}
	wg.Wait()

	row, err := dao.NewAPI4ConversationDAO().GetBySessionID(ctx, dao.DB, "sess-p3", "dlg-p1", true, dao.ConversationHistoryOptions{IncludeReference: true})
	if err != nil || row == nil {
		t.Fatalf("re-read session: row=%v err=%v", row, err)
	}
	if turns := parseChatbotTurns(row.Message); len(turns) != 4 {
		t.Fatalf("want both exchanges persisted (4 turns), got %d: %+v", len(turns), turns)
	}
	var refs []map[string]any
	for _, message := range parseChatbotTurns(row.Message) {
		if reference, ok := message["reference"].(map[string]any); ok {
			refs = append(refs, reference)
		}
	}
	if len(refs) != 2 {
		t.Fatalf("want 2 reference entries, got %d: %+v", len(refs), refs)
	}
}

// recordingResponseWriter is a minimal http.ResponseWriter stub
// for SSE frame tests. Tracks writes so the test can assert the
// emitted frame contents.
type recordingResponseWriter struct {
	header http.Header
	body   bytes.Buffer
}

func (r *recordingResponseWriter) Header() http.Header {
	return r.header
}
func (r *recordingResponseWriter) Write(b []byte) (int, error) {
	return r.body.Write(b)
}
func (r *recordingResponseWriter) WriteHeader(_ int) {}

// seedStreamTurnSession creates a persisted api_4_conversation row the
// streamChatbotTurn tests can append to.
func seedStreamTurnSession(t *testing.T, id, question, messageID string) {
	t.Helper()
	raw, err := json.Marshal([]map[string]any{{"role": "user", "content": question, "id": messageID, "created_at": 1}})
	if err != nil {
		t.Fatal(err)
	}
	sess := &entity.API4Conversation{ID: id, DialogID: "dlg-s1", UserID: "tenant-1", Message: raw}
	if err := dao.NewAPI4ConversationDAO().Create(t.Context(), dao.DB, sess); err != nil {
		t.Fatalf("seed session: %v", err)
	}
}

// TestStreamChatbotTurn_FinalFrameAndPersistenceUseRawStreamText pins
// the python iframe wire contract for the final SSE frame and the
// persisted turn:
//
//   - The final frame carries the retrieval reference but an empty
//     `answer` whenever text was already streamed as deltas (python
//     async_chat sets final["answer"] = "").
//   - The persisted assistant turn is the raw streamed text, NOT the
//     decorated final answer. Persisting the decorated text leaks the
//     server-inserted [ID:n] markers into the next turn's prompt
//     history; models that imitate the history format then emit
//     citation markers on turns whose retrieval returned nothing, and
//     the widget renders those as "Reference unavailable" icons.
func TestStreamChatbotTurn_FinalFrameAndPersistenceUseRawStreamText(t *testing.T) {
	db := setupServiceTestDB(t)
	pushServiceDB(t, db)

	seedStreamTurnSession(t, "sess-s1", "q1", "msg-s1")
	svc := NewBotService(nil)

	ref := map[string]any{
		"chunks":   []any{map[string]any{"chunk_id": "c1"}, map[string]any{"chunk_id": "c2"}},
		"doc_aggs": []any{},
	}
	results := make(chan AsyncChatResult, 5)
	results <- AsyncChatResult{Answer: "答案", Reference: map[string]any{}}
	results <- AsyncChatResult{Answer: "第一段", Reference: map[string]any{}}
	// Decorated final answer with server-inserted markers — must not
	// reach the wire (deltas exist) nor the persisted history.
	results <- AsyncChatResult{Answer: "答案第一段 [ID:0][ID:1]", Reference: ref, Final: true}
	close(results)

	sess := &entity.API4Conversation{ID: "sess-s1", DialogID: "dlg-s1", UserID: "tenant-1"}
	var frames []ChatbotSSEFrame
	for f := range svc.streamChatbotTurn(t.Context(), sess, "q1", "msg-s1", results) {
		frames = append(frames, f)
	}

	if len(frames) != 4 || !frames[3].Done {
		t.Fatalf("want 3 frames + done, got %+v", frames)
	}
	final := frames[2]
	if !final.Final {
		t.Fatalf("frame 2 must be final: %+v", final)
	}
	if final.Data != "" {
		t.Errorf("final frame answer = %q, want empty (deltas already streamed)", final.Data)
	}
	if !reflect.DeepEqual(final.Reference, ref) {
		t.Errorf("final frame reference = %+v, want the retrieval reference", final.Reference)
	}
	if frames[0].Data != "答案" || frames[1].Data != "第一段" {
		t.Errorf("delta frames must forward their own delta: %+v %+v", frames[0], frames[1])
	}

	row, err := dao.NewAPI4ConversationDAO().GetBySessionID(t.Context(), dao.DB, "sess-s1", "dlg-s1", true)
	if err != nil || row == nil {
		t.Fatalf("re-read session: row=%v err=%v", row, err)
	}
	turns := parseChatbotTurns(row.Message)
	if len(turns) != 2 || turns[1]["role"] != "assistant" {
		t.Fatalf("want user+assistant pair, got %+v", turns)
	}
	if got := turns[1]["content"]; got != "答案第一段" {
		t.Errorf("persisted assistant content = %v, want raw streamed text without server-inserted markers", got)
	}
}

// TestStreamChatbotTurn_ThinkMarkersPersistedAsTags ensures the raw
// stream text keeps the <think>/</think> tags the marker frames stand
// for, matching python structure_answer's accumulation.
func TestStreamChatbotTurn_ThinkMarkersPersistedAsTags(t *testing.T) {
	db := setupServiceTestDB(t)
	pushServiceDB(t, db)

	seedStreamTurnSession(t, "sess-s2", "q2", "msg-s2")
	svc := NewBotService(nil)

	results := make(chan AsyncChatResult, 5)
	results <- AsyncChatResult{StartToThink: true}
	results <- AsyncChatResult{Answer: "思考中", Reference: map[string]any{}}
	results <- AsyncChatResult{EndToThink: true}
	results <- AsyncChatResult{Answer: "结论", Reference: map[string]any{}}
	results <- AsyncChatResult{Answer: "<think>思考中</think>结论", Final: true}
	close(results)

	sess := &entity.API4Conversation{ID: "sess-s2", DialogID: "dlg-s1", UserID: "tenant-1"}
	for range svc.streamChatbotTurn(t.Context(), sess, "q2", "msg-s2", results) {
	}

	row, err := dao.NewAPI4ConversationDAO().GetBySessionID(t.Context(), dao.DB, "sess-s2", "dlg-s1", true)
	if err != nil || row == nil {
		t.Fatalf("re-read session: row=%v err=%v", row, err)
	}
	turns := parseChatbotTurns(row.Message)
	if len(turns) != 2 {
		t.Fatalf("want user+assistant pair, got %+v", turns)
	}
	if got := turns[1]["content"]; got != "<think>思考中</think>结论" {
		t.Errorf("persisted assistant content = %v, want <think>思考中</think>结论", got)
	}
}

// TestStreamChatbotTurn_SingleShotFinalKeepsText covers finals that
// arrive without any streamed delta (e.g. the structured-SQL path):
// the final frame is the only carrier of the text and must keep it,
// and it is what gets persisted.
func TestStreamChatbotTurn_SingleShotFinalKeepsText(t *testing.T) {
	db := setupServiceTestDB(t)
	pushServiceDB(t, db)

	seedStreamTurnSession(t, "sess-s3", "q3", "msg-s3")
	svc := NewBotService(nil)

	ref := map[string]any{"chunks": []any{}, "doc_aggs": []any{}}
	results := make(chan AsyncChatResult, 1)
	results <- AsyncChatResult{Answer: "sql result", Reference: ref, Final: true}
	close(results)

	sess := &entity.API4Conversation{ID: "sess-s3", DialogID: "dlg-s1", UserID: "tenant-1"}
	var frames []ChatbotSSEFrame
	for f := range svc.streamChatbotTurn(t.Context(), sess, "q3", "msg-s3", results) {
		frames = append(frames, f)
	}
	if len(frames) != 2 || !frames[1].Done {
		t.Fatalf("want final + done, got %+v", frames)
	}
	if frames[0].Data != "sql result" {
		t.Errorf("single-shot final must carry the text, got %q", frames[0].Data)
	}

	row, err := dao.NewAPI4ConversationDAO().GetBySessionID(t.Context(), dao.DB, "sess-s3", "dlg-s1", true)
	if err != nil || row == nil {
		t.Fatalf("re-read session: row=%v err=%v", row, err)
	}
	turns := parseChatbotTurns(row.Message)
	if len(turns) != 2 || turns[1]["content"] != "sql result" {
		t.Errorf("persisted turns = %+v, want assistant content 'sql result'", turns)
	}
}

// TestStreamChatbotTurn_ErrorKeptOnWireAndNotPersisted mirrors the
// python exception path: the error text must stay visible on the wire
// (it was never streamed as a delta) but nothing is persisted.
func TestStreamChatbotTurn_ErrorKeptOnWireAndNotPersisted(t *testing.T) {
	db := setupServiceTestDB(t)
	pushServiceDB(t, db)

	seedStreamTurnSession(t, "sess-s4", "q4", "msg-s4")
	svc := NewBotService(nil)

	results := make(chan AsyncChatResult, 1)
	results <- AsyncChatResult{Answer: "**ERROR**: boom", Final: true}
	close(results)

	sess := &entity.API4Conversation{ID: "sess-s4", DialogID: "dlg-s1", UserID: "tenant-1"}
	var frames []ChatbotSSEFrame
	for f := range svc.streamChatbotTurn(t.Context(), sess, "q4", "msg-s4", results) {
		frames = append(frames, f)
	}
	if len(frames) != 2 || !frames[1].Done {
		t.Fatalf("want final + done, got %+v", frames)
	}
	if frames[0].Data != "**ERROR**: boom" {
		t.Errorf("error final must carry the text, got %q", frames[0].Data)
	}

	row, err := dao.NewAPI4ConversationDAO().GetBySessionID(t.Context(), dao.DB, "sess-s4", "dlg-s1", true)
	if err != nil || row == nil {
		t.Fatalf("re-read session: row=%v err=%v", row, err)
	}
	if turns := parseChatbotTurns(row.Message); len(turns) != 1 || turns[0]["role"] != "user" {
		t.Errorf("error must retain only the user question, got %+v", turns)
	}
}

// TestStreamChatbotTurn_ErrorAfterDeltasKeptOnWire covers a pipeline-level
// failure that arrives AFTER partial deltas were already streamed
// (chat_pipeline.go reports a mid-stream driver error as a final
// "**ERROR**" result): the final frame must still carry the error text —
// dropping it would leave the client with a silently truncated partial
// answer — and nothing is persisted.
func TestStreamChatbotTurn_ErrorAfterDeltasKeptOnWire(t *testing.T) {
	db := setupServiceTestDB(t)
	pushServiceDB(t, db)

	seedStreamTurnSession(t, "sess-s5", "q5", "msg-s5")
	svc := NewBotService(nil)

	results := make(chan AsyncChatResult, 2)
	results <- AsyncChatResult{Answer: "partial", Reference: map[string]any{}}
	results <- AsyncChatResult{Answer: "**ERROR**: boom", Final: true}
	close(results)

	sess := &entity.API4Conversation{ID: "sess-s5", DialogID: "dlg-s1", UserID: "tenant-1"}
	var frames []ChatbotSSEFrame
	for f := range svc.streamChatbotTurn(t.Context(), sess, "q5", "msg-s5", results) {
		frames = append(frames, f)
	}
	if len(frames) != 3 || !frames[2].Done {
		t.Fatalf("want delta + final + done, got %+v", frames)
	}
	if frames[0].Data != "partial" {
		t.Errorf("delta frame = %q, want partial", frames[0].Data)
	}
	if !frames[1].Final || frames[1].Data != "**ERROR**: boom" {
		t.Errorf("error final after deltas must carry the error text, got %+v", frames[1])
	}

	row, err := dao.NewAPI4ConversationDAO().GetBySessionID(t.Context(), dao.DB, "sess-s5", "dlg-s1", true)
	if err != nil || row == nil {
		t.Fatalf("re-read session: row=%v err=%v", row, err)
	}
	if turns := parseChatbotTurns(row.Message); len(turns) != 1 || turns[0]["role"] != "user" {
		t.Errorf("error must retain only the user question, got %+v", turns)
	}
}

// TestStreamChatbotTurn_ReasoningFieldForwardedAsStreamText covers the
// tool-path delivery mode: chat_pipeline.go's ChatStreamlyWithTools
// callback routes in-think text through the Reasoning field (so the
// OpenAI-compat SSE handler can map it to delta.reasoning_content)
// instead of Answer deltas. Python delivers the same reasoning as
// <think>-wrapped answer stream text (rag/llm/chat_model.py), so the
// iframe wire and the persisted history must carry it too — dropping it
// would leave the widget's think block empty.
func TestStreamChatbotTurn_ReasoningFieldForwardedAsStreamText(t *testing.T) {
	db := setupServiceTestDB(t)
	pushServiceDB(t, db)

	seedStreamTurnSession(t, "sess-s6", "q6", "msg-s6")
	svc := NewBotService(nil)

	results := make(chan AsyncChatResult, 5)
	results <- AsyncChatResult{StartToThink: true}
	results <- AsyncChatResult{Reasoning: "推理中", Reference: map[string]any{}}
	results <- AsyncChatResult{EndToThink: true}
	results <- AsyncChatResult{Answer: "结论", Reference: map[string]any{}}
	results <- AsyncChatResult{Answer: "<think>推理中</think>结论", Final: true}
	close(results)

	sess := &entity.API4Conversation{ID: "sess-s6", DialogID: "dlg-s1", UserID: "tenant-1"}
	var frames []ChatbotSSEFrame
	for f := range svc.streamChatbotTurn(t.Context(), sess, "q6", "msg-s6", results) {
		frames = append(frames, f)
	}
	if len(frames) != 6 || !frames[5].Done {
		t.Fatalf("want think + reasoning + think + delta + final + done, got %+v", frames)
	}
	if frames[1].Data != "推理中" {
		t.Errorf("reasoning frame must forward the reasoning text, got %+v", frames[1])
	}
	if frames[4].Final && frames[4].Data != "" {
		t.Errorf("final frame answer = %q, want empty (deltas already streamed)", frames[4].Data)
	}

	row, err := dao.NewAPI4ConversationDAO().GetBySessionID(t.Context(), dao.DB, "sess-s6", "dlg-s1", true)
	if err != nil || row == nil {
		t.Fatalf("re-read session: row=%v err=%v", row, err)
	}
	turns := parseChatbotTurns(row.Message)
	if len(turns) != 2 {
		t.Fatalf("want user+assistant pair, got %+v", turns)
	}
	if got := turns[1]["content"]; got != "<think>推理中</think>结论" {
		t.Errorf("persisted assistant content = %v, want <think>推理中</think>结论", got)
	}
}

func TestStreamChatbotTurn_ContextCancellationUnblocksFullChannel(t *testing.T) {
	db := setupServiceTestDB(t)
	pushServiceDB(t, db)

	seedStreamTurnSession(t, "sess-cancel-leak", "q_leak", "msg-leak")
	svc := NewBotService(nil)

	ctx, cancel := context.WithCancel(context.Background())

	results := make(chan AsyncChatResult, 30)
	for i := 0; i < 30; i++ {
		results <- AsyncChatResult{Answer: fmt.Sprintf("chunk-%d", i)}
	}
	close(results)

	sess := &entity.API4Conversation{ID: "sess-cancel-leak", DialogID: "dlg-s1", UserID: "tenant-1"}

	// Cancel context to simulate client disconnect while channel buffer is full
	cancel()

	frames := svc.streamChatbotTurn(ctx, sess, "q_leak", "msg-leak", results)

	// Do NOT drain concurrently. streamChatbotTurn should detect ctx.Done()
	// and close frames promptly.
	timeout := time.After(2 * time.Second)
	for {
		select {
		case _, ok := <-frames:
			if !ok {
				goto verified
			}
		case <-timeout:
			t.Fatal("streamChatbotTurn leaked: did not terminate when context was cancelled")
		}
	}

verified:
	// The user question is preserved in history; failure/cancellation leaves it intact.
	row, err := dao.NewAPI4ConversationDAO().GetBySessionID(context.Background(), dao.DB, "sess-cancel-leak", "dlg-s1", true)
	if err != nil {
		t.Fatalf("re-read session: %v", err)
	}
	if row != nil {
		turns := parseChatbotTurns(row.Message)
		if len(turns) != 1 || turns[0]["id"] != "msg-leak" {
			t.Fatalf("expected user turn to be preserved in history, got: %+v", turns)
		}
	}
}