Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

refactor(common): unify </think>-strip into shared StripThinkTrailing helper (#18179)

jay777210a4cc65db04c
diff --git a/internal/agent/component/docs_generator.go b/internal/agent/component/docs_generator.go
index ce01f52441..cdab0df9af 100644
--- a/internal/agent/component/docs_generator.go
+++ b/internal/agent/component/docs_generator.go
@@ -43,6 +43,7 @@ import (
iow "ragflow/internal/agent/component/io"
"ragflow/internal/agent/runtime"
+ "ragflow/internal/common"
"ragflow/internal/storage"
"ragflow/internal/utility"
)
@@ -349,7 +350,11 @@ func resolveDocsGeneratorContent(ctx context.Context, configured string, inputs
}
}
if !strings.Contains(content, "{{") {
- return stripThinking(content), nil
+ // The think cut itself is the greedy re.sub(r"^.*</think>", "", s,
+ // re.DOTALL) pattern (as in generator.py). The unconditional trim —
+ // applied even when no think block is present — mirrors Python's
+ // _strip_thinking, which always .strip()s the exported content.
+ return strings.TrimSpace(common.StripThinkTrailing(content)), nil
}
state, _, err := runtime.GetStateFromContext[*runtime.CanvasState](ctx)
@@ -363,14 +368,7 @@ func resolveDocsGeneratorContent(ctx context.Context, configured string, inputs
if err != nil {
return "", fmt.Errorf("resolve content template: %w", err)
}
- return stripThinking(resolved), nil
-}
-
-func stripThinking(content string) string {
- if idx := strings.LastIndex(content, "</think>"); idx >= 0 {
- return strings.TrimSpace(content[idx+len("</think>"):])
- }
- return content
+ return strings.TrimSpace(common.StripThinkTrailing(resolved)), nil
}
func storeAgentAttachment(ctx context.Context, docID string, payload []byte) (bool, error) {
diff --git a/internal/agent/component/docs_generator_test.go b/internal/agent/component/docs_generator_test.go
index 60511ca1c3..a460cca662 100644
--- a/internal/agent/component/docs_generator_test.go
+++ b/internal/agent/component/docs_generator_test.go
@@ -227,3 +227,24 @@ func TestDocsGenerator_Invoke_StoresAgentAttachment(t *testing.T) {
t.Fatalf("download_info has type %T, want map", out["download_info"])
}
}
+
+// TestResolveDocsGeneratorContent_ThinkAndTrim pins the static-content path:
+// a think preamble is stripped, and outer whitespace is trimmed even when no
+// think block is present — matching Python _strip_thinking's final .strip().
+func TestResolveDocsGeneratorContent_ThinkAndTrim(t *testing.T) {
+ got, err := resolveDocsGeneratorContent(t.Context(), "<think>reasoning</think>\n# Title\n", nil)
+ if err != nil {
+ t.Fatalf("resolveDocsGeneratorContent: %v", err)
+ }
+ if got != "# Title" {
+ t.Errorf("content = %q, want %q", got, "# Title")
+ }
+
+ got, err = resolveDocsGeneratorContent(t.Context(), " plain \n", nil)
+ if err != nil {
+ t.Fatalf("resolveDocsGeneratorContent: %v", err)
+ }
+ if got != "plain" {
+ t.Errorf("content = %q, want %q", got, "plain")
+ }
+}
diff --git a/internal/agent/component/llm.go b/internal/agent/component/llm.go
index e2ee884918..f06a43a8b4 100644
--- a/internal/agent/component/llm.go
+++ b/internal/agent/component/llm.go
@@ -1300,13 +1300,12 @@ func init() {
// This removes DeepSeek-R1-style thinking blocks and JSON-fence
// prefixes/suffixes from the raw model response.
var (
- reThinkPrefix = regexp.MustCompile(`(?s)^.*</think>`)
reJSONFencePrefix = regexp.MustCompile(`(?s)^.*` + "```json")
reJSONFenceSuffix = regexp.MustCompile("```\n*$")
)
func cleanFormattedAnswer(ans string) string {
- ans = reThinkPrefix.ReplaceAllString(ans, "")
+ ans = common.StripThinkTrailing(ans)
ans = reJSONFencePrefix.ReplaceAllString(ans, "")
ans = reJSONFenceSuffix.ReplaceAllString(ans, "")
return ans
diff --git a/internal/agent/component/llm_test.go b/internal/agent/component/llm_test.go
index 79657751ca..c84a5887c1 100644
--- a/internal/agent/component/llm_test.go
+++ b/internal/agent/component/llm_test.go
@@ -799,3 +799,26 @@ func TestLLM_Invoke_UsesModelContentLengthBudget(t *testing.T) {
t.Fatal("user prompt was modified by fitting despite fitting the content_length budget")
}
}
+
+// TestCleanFormattedAnswer pins cleanFormattedAnswer's pipeline: think-block
+// strip (common.StripThinkTrailing) first, then JSON-fence prefix/suffix.
+func TestCleanFormattedAnswer(t *testing.T) {
+ tests := []struct {
+ name string
+ in string
+ want string
+ }{
+ {name: "plain", in: "plain answer", want: "plain answer"},
+ {name: "think prefix", in: "<think>reasoning</think>{\"a\":1}", want: "{\"a\":1}"},
+ {name: "mid-text think", in: "note<think>reasoning</think>{\"a\":1}", want: "{\"a\":1}"},
+ {name: "json fence", in: "```json\n{\"a\":1}\n```", want: "\n{\"a\":1}\n"},
+ {name: "think then fence", in: "<think>reasoning</think>```json\n{\"a\":1}\n```", want: "\n{\"a\":1}\n"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := cleanFormattedAnswer(tt.in); got != tt.want {
+ t.Errorf("cleanFormattedAnswer(%q) = %q, want %q", tt.in, got, tt.want)
+ }
+ })
+ }
+}
diff --git a/internal/agent/harness/route.go b/internal/agent/harness/route.go
index dff1219b78..f6af095b87 100644
--- a/internal/agent/harness/route.go
+++ b/internal/agent/harness/route.go
@@ -26,6 +26,7 @@ import (
"github.com/cloudwego/eino/schema"
"ragflow/internal/agent/chat"
+ "ragflow/internal/common"
"gorm.io/gorm"
)
@@ -135,14 +136,13 @@ func fallbackRoute(question, modeLabel, reason string) RouteDecision {
}
var (
- reThinkTag = regexp.MustCompile(`(?s)^.*</think>`)
- reFence = regexp.MustCompile("```(?:json)?\\s*|\\s*```")
+ reFence = regexp.MustCompile("```(?:json)?\\s*|\\s*```")
)
// unmarshalModelJSON mirrors Python's _extract_json: strip thinking preamble and
// Markdown fences, then parse JSON.
func unmarshalModelJSON(text string, out interface{}) error {
- text = reThinkTag.ReplaceAllString(text, "")
+ text = common.StripThinkTrailing(text)
text = reFence.ReplaceAllString(text, "")
text = strings.TrimSpace(text)
if text == "" {
diff --git a/internal/agent/harness/route_test.go b/internal/agent/harness/route_test.go
index da9eec04a7..7d78a57ab2 100644
--- a/internal/agent/harness/route_test.go
+++ b/internal/agent/harness/route_test.go
@@ -2,6 +2,7 @@ package harness
import (
"context"
+ "reflect"
"testing"
"gorm.io/gorm"
@@ -180,3 +181,30 @@ func TestPlannerNode_BadJSONFallsBack(t *testing.T) {
t.Fatalf("fallback plan = %+v, want direct", plan)
}
}
+
+// TestUnmarshalModelJSON pins unmarshalModelJSON: think preamble and Markdown
+// fences are stripped before JSON parsing, and an empty payload yields {}.
+func TestUnmarshalModelJSON(t *testing.T) {
+ tests := []struct {
+ name string
+ in string
+ want map[string]any
+ }{
+ {name: "plain json", in: `{"answer":"42"}`, want: map[string]any{"answer": "42"}},
+ {name: "think prefix", in: `<think>reason</think>{"answer":"42"}`, want: map[string]any{"answer": "42"}},
+ {name: "json fence", in: "```json\n{\"answer\":\"42\"}\n```", want: map[string]any{"answer": "42"}},
+ {name: "think then fence", in: "<think>r</think>```json\n{\"answer\":\"42\"}\n```", want: map[string]any{"answer": "42"}},
+ {name: "empty becomes empty object", in: "", want: map[string]any{}},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ var out map[string]any
+ if err := unmarshalModelJSON(tt.in, &out); err != nil {
+ t.Fatalf("unmarshalModelJSON(%q): %v", tt.in, err)
+ }
+ if !reflect.DeepEqual(out, tt.want) {
+ t.Errorf("unmarshalModelJSON(%q) = %#v, want %#v", tt.in, out, tt.want)
+ }
+ })
+ }
+}
diff --git a/internal/common/think.go b/internal/common/think.go
new file mode 100644
index 0000000000..577908a74a
--- /dev/null
+++ b/internal/common/think.go
@@ -0,0 +1,23 @@
+package common
+
+import "strings"
+
+// StripThinkTrailing cuts everything through the LAST `</think>` marker on the
+// input and returns what follows, mirroring Python's
+//
+// re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
+//
+// The Python regex is greedy: with re.DOTALL the leading `.*` matches as much
+// as possible, so for an input with more than one `</think>` everything up to
+// and including the LAST marker is stripped. A non-greedy form would strip only
+// the first marker and leave a residual think block in the response. Go's
+// strings.LastIndex has the same greedy semantics, so it is used instead of a
+// regex. When no `</think>` is present, s is returned unchanged. Note the marker
+// is matched regardless of whether an opening `<think>` exists — parity with
+// Python, which does not require one.
+func StripThinkTrailing(s string) string {
+ if i := strings.LastIndex(s, "</think>"); i >= 0 {
+ return s[i+len("</think>"):]
+ }
+ return s
+}
diff --git a/internal/common/think_test.go b/internal/common/think_test.go
new file mode 100644
index 0000000000..4e26a67afb
--- /dev/null
+++ b/internal/common/think_test.go
@@ -0,0 +1,75 @@
+package common
+
+import "testing"
+
+// TestStripThinkTrailingParityWithPython pins down the exact behaviour of
+// StripThinkTrailing so it stays in sync with the Python original
+//
+// re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
+//
+// The Python regex is greedy: with re.DOTALL the leading `.*` matches as much
+// as possible, so for an input with more than one </think> the substitution
+// strips everything up to and including the LAST marker. A non-greedy `*?`
+// would diverge here and leave the tail-visible-portion of the response behind.
+func TestStripThinkTrailingParityWithPython(t *testing.T) {
+ tests := []struct {
+ name string
+ in string
+ want string
+ }{
+ {
+ name: "no_think_tag_unchanged",
+ in: "plain answer, no tags",
+ want: "plain answer, no tags",
+ },
+ {
+ name: "single_think_block_stripped",
+ in: "<think>hidden reasoning</think>visible answer",
+ want: "visible answer",
+ },
+ {
+ name: "multiline_single_think_block",
+ in: "<think>\nline 1\nline 2\n</think>\nvisible",
+ want: "\nvisible",
+ },
+ {
+ name: "two_think_blocks_greedy_strips_to_last",
+ // Python: greedy `^.*</think>` strips
+ // "<think>A</think>part1<think>B</think>", leaving "part2".
+ // A non-greedy form would have left "part1<think>B</think>part2".
+ in: "<think>A</think>part1<think>B</think>part2",
+ want: "part2",
+ },
+ {
+ name: "two_think_blocks_with_answer_greedy",
+ // Mirrors a real-world malformed stream where the model
+ // re-emits a stray </think> after the answer.
+ in: "<think>reasoning</think>Answer<think>noise</think>real tail",
+ want: "real tail",
+ },
+ {
+ name: "unclosed_think_tag_does_not_match",
+ // No </think> at all — the helper requires the closing tag,
+ // so nothing is stripped and the original passes through.
+ in: "<think>no closing tag here",
+ want: "<think>no closing tag here",
+ },
+ {
+ name: "lookalike_closing_tag_strips_everything",
+ // Quirky but intentional parity case: the helper only requires
+ // the substring </think> to be present, regardless of whether
+ // an opening <think> exists. The Python original has the same
+ // behaviour, so the port must match — this is a pre-existing
+ // limitation, not a new bug.
+ in: "use </tag> to mean end, not the same as </think>",
+ want: "",
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := StripThinkTrailing(tt.in); got != tt.want {
+ t.Errorf("input: %q\n got: %q\n want: %q", tt.in, got, tt.want)
+ }
+ })
+ }
+}
diff --git a/internal/ingestion/component/extractor.go b/internal/ingestion/component/extractor.go
index 64679dc1bd..01a5f92d54 100644
--- a/internal/ingestion/component/extractor.go
+++ b/internal/ingestion/component/extractor.go
@@ -1035,29 +1035,16 @@ var toolCallRE = regexp.MustCompile(`(?s)<tool_call>.*?</tool_call>`)
// to parse explicitly.
func cleanLLMText(s string) string {
if strings.HasPrefix(s, "<think>") {
- if j := strings.LastIndex(s, "</think>"); j >= 0 {
- s = s[j+len("</think>"):]
- }
+ s = common.StripThinkTrailing(s)
}
s = toolCallRE.ReplaceAllString(s, "")
return strings.TrimSpace(s)
}
-// stripTrailingThink mirrors Python's re.sub(r"^.*</think>", "", s, re.DOTALL)
-// in generator.py's post-processing (e.g. gen_metadata:960): cut everything
-// through the LAST `</think>` so a mid-text reasoning block preceded by a
-// preamble is stripped on top of cleanLLMText's leading-only cleanup.
-func stripTrailingThink(s string) string {
- if i := strings.LastIndex(s, "</think>"); i >= 0 {
- s = s[i+len("</think>"):]
- }
- return s
-}
-
// cleanExtractionResult strips `</think>` tags and rejects `**ERROR**` responses,
// matching Python's keyword_extraction and question_proposal post-processing.
func cleanExtractionResult(s string) string {
- s = stripTrailingThink(s)
+ s = common.StripThinkTrailing(s)
s = strings.TrimSpace(s)
if strings.Contains(s, "**ERROR**") {
return ""
@@ -1214,7 +1201,7 @@ func (c *ExtractorComponent) callStructured(ctx context.Context, db *gorm.DB, in
// would otherwise survive into JSON parsing and silently drop the whole
// metadata extraction that Python would have kept. No **ERROR** check here
// — Python's gen_metadata has none either.
- s = stripTrailingThink(s)
+ s = common.StripThinkTrailing(s)
parsed, ok := tryParseJSONObject(s)
if !ok {
return nil, nil
diff --git a/internal/ingestion/component/extractor_tag.go b/internal/ingestion/component/extractor_tag.go
index 6d4dad1235..a84b759a41 100644
--- a/internal/ingestion/component/extractor_tag.go
+++ b/internal/ingestion/component/extractor_tag.go
@@ -739,10 +739,7 @@ func buildTaggerPrompt(topN int, tagSetStr string, examples []schema.TaggedChunk
}
func parseTaggerResponse(raw string, topN int) map[string]int {
- raw = strings.TrimSpace(raw)
- if idx := strings.LastIndex(raw, "</think>"); idx >= 0 {
- raw = strings.TrimSpace(raw[idx+len("</think>"):])
- }
+ raw = strings.TrimSpace(common.StripThinkTrailing(raw))
if strings.Contains(raw, "**ERROR**") {
common.Warn("extractor tags: LLM returned **ERROR**")
return nil
diff --git a/internal/ingestion/component/extractor_test.go b/internal/ingestion/component/extractor_test.go
index e83688f5f8..8e036f9015 100644
--- a/internal/ingestion/component/extractor_test.go
+++ b/internal/ingestion/component/extractor_test.go
@@ -905,7 +905,7 @@ func TestExtractorComponent_runEnableMetadata_StripsJSONFence(t *testing.T) {
// metadata path — LLM call, second-layer <think> strip, JSON parse, merge —
// tolerates a mid-text reasoning block preceded by a preamble, matching
// Python gen_metadata. This is the end-to-end guard for callStructured's
-// stripTrailingThink: without it the metadata extraction silently drops.
+// common.StripThinkTrailing: without it the metadata extraction silently drops.
func TestExtractorComponent_runEnableMetadata_MidTextThink(t *testing.T) {
withStubChatInvoker(t, stubResponse{Content: `preamble<think>reasoning</think>{"category":"finance"}`})
c := newMetadataExtractor(common.MetadataFieldDef{Key: "category", Type: "string"})
@@ -1375,7 +1375,7 @@ func TestExtractorComponent_callStructured(t *testing.T) {
}
// TestExtractorComponent_callStructured_MidTextThink verifies the metadata
-// path's second cleanup layer (stripTrailingThink) strips a mid-text
+// path's second cleanup layer (common.StripThinkTrailing) strips a mid-text
// reasoning block preceded by a preamble, matching Python's gen_metadata
// double cleanup (async_chat + re.sub r"^.*</think>"). Without it the JSON
// would survive the leading-only cleanLLMText, fail to parse, and silently
@@ -1392,30 +1392,6 @@ func TestExtractorComponent_callStructured_MidTextThink(t *testing.T) {
}
}
-// TestStripTrailingThink verifies the helper mirrors Python's
-// re.sub(r"^.*</think>", "", s, re.DOTALL): it cuts through the LAST
-// </think> and never inspects for **ERROR**.
-func TestStripTrailingThink(t *testing.T) {
- tests := []struct {
- name string
- in string
- want string
- }{
- {name: "mid-text think", in: `prefix<think>r</think>{"a":1}`, want: `{"a":1}`},
- {name: "no think", in: "plain", want: "plain"},
- {name: "multiple closes", in: "<think>a</think>mid<think>b</think>tail", want: "tail"},
- {name: "close without open", in: "abc</think>def", want: "def"},
- {name: "open without close", in: "<think>unclosed", want: "<think>unclosed"},
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- if got := stripTrailingThink(tt.in); got != tt.want {
- t.Errorf("stripTrailingThink(%q) = %q, want %q", tt.in, got, tt.want)
- }
- })
- }
-}
-
// TestExtractorComponent_Invoke_ConcurrentKeywordsAndQuestions verifies
// that when both auto_keywords and auto_questions are enabled, both
// LLM calls are dispatched per chunk and results land on the chunk
diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor.go b/internal/ingestion/component/knowledge_compiler/tree/raptor.go
index 486d2bdb2f..9cab38cb48 100644
--- a/internal/ingestion/component/knowledge_compiler/tree/raptor.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor.go
@@ -21,6 +21,7 @@ import (
"strings"
"time"
+ rfcommon "ragflow/internal/common"
"ragflow/internal/ingestion/component/knowledge_compiler/common"
"ragflow/internal/tokenizer"
)
@@ -369,9 +370,12 @@ func summarizeTexts(ctx context.Context, deps common.Deps, llmID, systemText, us
}
content := resp.Content
// Strip reasoning preamble up to the final thinking close tag
- // (Python: re.sub(r"^.*</think>", "", response, DOTALL)).
- if i := strings.LastIndex(content, "</think>"); i >= 0 {
- content = content[i+len("</think>"):]
+ // (Python: re.sub(r"^.*</think>", "", response, DOTALL)). The
+ // else-if keeps the vendor-specific </think:6124c78e> fallback
+ // active only when no standard </think> is present, preserving
+ // the original control flow.
+ if strings.Contains(content, "</think>") {
+ content = rfcommon.StripThinkTrailing(content)
} else if i := strings.LastIndex(content, "</think:6124c78e>"); i >= 0 {
content = content[i+len("</think:6124c78e>"):]
}
diff --git a/internal/ingestion/component/knowledge_compiler/tree/raptor_test.go b/internal/ingestion/component/knowledge_compiler/tree/raptor_test.go
index 67a09f3078..c95ed71624 100644
--- a/internal/ingestion/component/knowledge_compiler/tree/raptor_test.go
+++ b/internal/ingestion/component/knowledge_compiler/tree/raptor_test.go
@@ -49,6 +49,30 @@ func TestSummarizeTextsStripsThinkPreamble(t *testing.T) {
}
}
+func TestSummarizeTextsStripsVendorThinkCloseFallback(t *testing.T) {
+ // The </think:6124c78e> fallback only runs when no standard </think>
+ // is present (else-if semantics). Both forms strip the preamble.
+ tests := []struct {
+ name string
+ content string
+ }{
+ {"vendor_close", "<think:6124c78e>reasoning</think:6124c78e>Vendor summary"},
+ {"standard_close", "<think>reasoning</think>Standard summary"},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ f := &fakeChat{responses: []*common.ChatResponse{{Content: tt.content}}}
+ got, err := summarizeTexts(context.Background(), depsWithChat(f), "llm", "sys", "user", 512)
+ if err != nil {
+ t.Fatalf("unexpected err: %v", err)
+ }
+ if strings.Contains(got, "</think") {
+ t.Fatalf("think close tag not stripped: %q", got)
+ }
+ })
+ }
+}
+
func TestSummarizeTextsStripsTruncationMarker(t *testing.T) {
marker := strings.Repeat("·", 6) + "\n由于长度的原因,回答被截断了,要继续吗?"
f := &fakeChat{responses: []*common.ChatResponse{{Content: "title\nbody " + marker}}}
diff --git a/internal/service/generator.go b/internal/service/generator.go
index d3e4b405a9..5789e4f35a 100644
--- a/internal/service/generator.go
+++ b/internal/service/generator.go
@@ -86,7 +86,7 @@ func KeywordExtraction(ctx context.Context, chatModel *modelModule.ChatModel, co
// Clean up response - remove thinking tags if present
result := strings.TrimSpace(*response.Answer)
- result = thinkBlockRE.ReplaceAllString(result, "")
+ result = common.StripThinkTrailing(result)
result = strings.TrimSpace(result)
if strings.Contains(result, "**ERROR**") {
@@ -188,7 +188,7 @@ func CrossLanguages(ctx context.Context, tenantID string, llmID string, query st
result := *response.Answer
// Clean up response - remove think tags and trim
- result = thinkBlockRE.ReplaceAllString(result, "")
+ result = common.StripThinkTrailing(result)
if strings.Contains(result, "**ERROR**") {
return query, nil
@@ -350,7 +350,7 @@ func FullQuestion(
return fallbackToLatestUser(messages), fmt.Errorf("FullQuestion: empty response")
}
cleaned := strings.TrimSpace(*resp.Answer)
- cleaned = thinkBlockRE.ReplaceAllString(cleaned, "")
+ cleaned = common.StripThinkTrailing(cleaned)
cleaned = strings.TrimSpace(cleaned)
if errorMarkerRE.MatchString(cleaned) {
return fallbackToLatestUser(messages), nil
diff --git a/internal/service/load_prompt.go b/internal/service/load_prompt.go
index 13df5570dd..c06c3a397a 100644
--- a/internal/service/load_prompt.go
+++ b/internal/service/load_prompt.go
@@ -33,22 +33,6 @@ var (
promptsBaseDir string
)
-// thinkBlockRE strips think blocks from LLM responses.
-//
-// This mirrors the Python original
-//
-// re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
-//
-// exactly: `.*` with re.DOTALL is greedy, so the regex consumes
-// everything from the start of the string up to and including the
-// LAST </think> on the input. The Go form uses [\s\S] (Go's
-// newline-aware any-char) and a greedy `*`. A non-greedy `*?` here
-// would diverge for responses containing more than one </think>
-// (e.g. malformed streams that re-emit a partial think block
-// after the real answer), stripping only the first and leaving the
-// rest of the response invisible to the caller.
-var thinkBlockRE = regexp.MustCompile(`^[\s\S]*</think>`)
-
// jsonFenceRE matches Markdown code fences around JSON responses.
// Mirrors Python's re.sub(r"(`{3}json\n|`{3}\n*$)", ..., flags=re.DOTALL).
// Note: `\n*` is intentionally narrower than Go's `\s*` — Python only
diff --git a/internal/service/load_prompt_test.go b/internal/service/load_prompt_test.go
index 81b0f32899..000711f015 100644
--- a/internal/service/load_prompt_test.go
+++ b/internal/service/load_prompt_test.go
@@ -18,23 +18,13 @@ package service
import "testing"
-// TestThinkBlockREParityWithPython pins down the exact behaviour of
-// thinkBlockRE so it stays in sync with the Python original
-//
-// re.sub(r"^.*</think>", "", ans, flags=re.DOTALL)
-//
-// The Python regex is greedy: with re.DOTALL the leading `.*` matches
-// as much as possible, so for an input with more than one </think>
-// the substitution strips everything up to and including the LAST
-// marker. A non-greedy `*?` would diverge here and leave the
-// tail-visible-portion of the response behind.
// TestJSONFenceREParityWithPython pins down jsonFenceRE against the Python
// original
//
// re.sub(r"(^.*</think>|```json\n|```\n*$)", "", ans, flags=re.DOTALL)
//
-// The Go port is split across thinkBlockRE (run first in callers) and
-// jsonFenceRE; this test exercises jsonFenceRE in isolation. The trailing
+// The Go port is split across common.StripThinkTrailing (run first in callers)
+// and jsonFenceRE; this test exercises jsonFenceRE in isolation. The trailing
// alternative uses \n* (not \s*) so a closing fence followed by other
// whitespace — e.g. "``` \n" — is preserved exactly as Python does.
func TestJSONFenceREParityWithPython(t *testing.T) {
@@ -114,67 +104,3 @@ func TestJSONFenceREParityWithPython(t *testing.T) {
})
}
}
-
-func TestThinkBlockREParityWithPython(t *testing.T) {
- tests := []struct {
- name string
- in string
- want string
- }{
- {
- name: "no_think_tag_unchanged",
- in: "plain answer, no tags",
- want: "plain answer, no tags",
- },
- {
- name: "single_think_block_stripped",
- in: "<think>hidden reasoning</think>visible answer",
- want: "visible answer",
- },
- {
- name: "multiline_single_think_block",
- in: "<think>\nline 1\nline 2\n</think>\nvisible",
- want: "\nvisible",
- },
- {
- name: "two_think_blocks_greedy_strips_to_last",
- // Python: greedy `^.*</think>` strips
- // "<think>A</think>part1<think>B</think>", leaving "part2".
- // A non-greedy form would have left "part1<think>B</think>part2".
- in: "<think>A</think>part1<think>B</think>part2",
- want: "part2",
- },
- {
- name: "two_think_blocks_with_answer_greedy",
- // Mirrors a real-world malformed stream where the model
- // re-emits a stray </think> after the answer.
- in: "<think>reasoning</think>Answer<think>noise</think>real tail",
- want: "real tail",
- },
- {
- name: "unclosed_think_tag_does_not_match",
- // No </think> at all — the regex requires the closing tag,
- // so nothing is stripped and the original passes through.
- in: "<think>no closing tag here",
- want: "<think>no closing tag here",
- },
- {
- name: "lookalike_closing_tag_strips_everything",
- // Quirky but intentional parity case: the regex only requires
- // the substring </think> to be present, regardless of whether
- // an opening <think> exists. The Python original has the same
- // behaviour, so the Go port must match — this is a
- // pre-existing limitation, not a new bug.
- in: "use </tag> to mean end, not the same as </think>",
- want: "",
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := thinkBlockRE.ReplaceAllString(tt.in, "")
- if got != tt.want {
- t.Errorf("input: %q\n got: %q\n want: %q", tt.in, got, tt.want)
- }
- })
- }
-}
diff --git a/internal/service/metadata_filter.go b/internal/service/metadata_filter.go
index cda02729ac..993f3d06b6 100644
--- a/internal/service/metadata_filter.go
+++ b/internal/service/metadata_filter.go
@@ -221,7 +221,7 @@ func GenMetaFilter(ctx context.Context, chatModel *modelModule.ChatModel, metaDa
// Clean up response
responseStr := strings.TrimSpace(*response.Answer)
- responseStr = thinkBlockRE.ReplaceAllString(responseStr, "")
+ responseStr = common.StripThinkTrailing(responseStr)
responseStr = jsonFenceRE.ReplaceAllString(responseStr, "")
responseStr = strings.TrimSpace(responseStr)
diff --git a/internal/service/think_tag.go b/internal/service/think_tag.go
index f0fdb71466..1a518d9dea 100644
--- a/internal/service/think_tag.go
+++ b/internal/service/think_tag.go
@@ -18,8 +18,10 @@ package service
import (
"context"
- "ragflow/internal/tokenizer"
"strings"
+
+ "ragflow/internal/common"
+ "ragflow/internal/tokenizer"
)
const thinkOpen = "<think>"
@@ -369,13 +371,10 @@ func ExtractVisibleAnswer(raw string) string {
if raw == "" {
return ""
}
- if !strings.Contains(raw, thinkClose) {
- return stripThinkTags(raw)
- }
-
- lastClose := strings.LastIndex(raw, thinkClose)
- answer := raw[lastClose+len(thinkClose):]
- return stripThinkTags(answer)
+ // Cut through the last </think> (common.StripThinkTrailing), then strip
+ // residual stray tags. With no </think> the cut is a no-op and all tags
+ // are stripped — same behaviour as before.
+ return stripThinkTags(common.StripThinkTrailing(raw))
}
// BufferAnswerDelta processes an answer delta through the think-state lifecycle.