Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(chat_pipeline): treat non-JSON model refusals as skipped graph extraction (#3606)

LuckVdb5c8d90cc207
diff --git a/internal/application/service/chat_pipeline/extract_entity.go b/internal/application/service/chat_pipeline/extract_entity.go
index e756fd5ca..deafb6d31 100644
--- a/internal/application/service/chat_pipeline/extract_entity.go
+++ b/internal/application/service/chat_pipeline/extract_entity.go
@@ -166,6 +166,34 @@ func (p *PluginExtractEntity) OnEvent(ctx context.Context,
// Graph extraction can return many nodes and relations; 4096 tokens can truncate the JSON payload.
const entityExtractionMaxTokens = 8192
+// ErrModelDeclined marks an LLM response that is prose rather than the
+// requested structured output — typically a refusal for chunks that carry
+// no extractable entities (table-of-contents pages, prompts with no content).
+// Retrying the same input cannot change the outcome (issue #3600), so the
+// async chunk-extraction task treats it as a skip. Other callers (the
+// /initialization "try extract" endpoint, query-side entity extraction)
+// keep reporting it as a failure.
+var ErrModelDeclined = errors.New("model output is prose, not extractable structured content")
+
+// previewDeclinedText truncates the model's prose for compact error logs.
+func previewDeclinedText(s string) string {
+ runes := []rune(strings.TrimSpace(s))
+ if len(runes) > 120 {
+ return string(runes[:120]) + "…"
+ }
+ return string(runes)
+}
+
+// modelStoppedOnItsOwn reports whether finish_reason proves the model ended
+// its answer by itself, which is the only case where a prose answer can be
+// read as a refusal. Budget exhaustion (every spelling IsLengthFinishReason
+// accepts), a stream that broke before the stop event ("incomplete"), and an
+// adapter that reports no reason at all are treated as possible truncation.
+func modelStoppedOnItsOwn(reason string) bool {
+ r := strings.ToLower(strings.TrimSpace(reason))
+ return r != "" && r != types.FinishReasonIncomplete && !IsLengthFinishReason(r)
+}
+
// Extractor is a struct for extracting entities
type Extractor struct {
chat chat.Chat
@@ -208,6 +236,14 @@ func (e *Extractor) Extract(ctx context.Context, content string) (*types.GraphDa
graph, err := e.formater.ParseGraph(ctx, chatResponse.Content)
if err != nil {
+ if errors.Is(err, ErrModelDeclined) && !modelStoppedOnItsOwn(chatResponse.FinishReason) {
+ // The output budget ran out (or the stream broke) before any JSON
+ // was written, e.g. the model spent it on prose "Step 1/Step 2"
+ // reasoning. That is truncation, not a refusal: keep it retriable.
+ // %v drops the ErrModelDeclined wrap on purpose.
+ err = fmt.Errorf("graph extraction stopped before JSON (finish_reason=%q): %v",
+ chatResponse.FinishReason, err)
+ }
logger.Errorf(ctx, "failed to parse graph: %v", err)
return nil, err
}
@@ -406,6 +442,17 @@ func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]i
err = json.Unmarshal([]byte(content), &parsed)
}
if err != nil {
+ // Distinguish "the model answered in prose" (e.g. it declined to
+ // extract a table-of-contents chunk with a Chinese "抱歉…" refusal)
+ // from "the model produced JSON that is malformed/truncated". A
+ // refusal contains no JSON structure at all; retrying the exact
+ // same input can never succeed, so callers treat it as a terminal
+ // skip instead of a retriable failure (issue #3600). Check the whole
+ // response, not just the extracted fence: a prose fence followed by a
+ // JSON one is a parse problem, not a refusal.
+ if !strings.ContainsAny(text, "{[") {
+ return nil, fmt.Errorf("%w: %s", ErrModelDeclined, previewDeclinedText(content))
+ }
return nil, fmt.Errorf("failed to parse %s content: %s", strings.ToUpper(string(f.formatType)), err.Error())
}
if parsed == nil {
@@ -435,6 +482,8 @@ func (f *Formater) parseOutput(ctx context.Context, text string) ([]map[string]i
func (f *Formater) ParseGraph(ctx context.Context, text string) (*types.GraphData, error) {
matchData, err := f.parseOutput(ctx, text)
if err != nil {
+ // A prose refusal comes back wrapped in ErrModelDeclined; the caller
+ // decides whether that is a skip (async chunk task) or a failure.
return nil, err
}
if len(matchData) == 0 {
diff --git a/internal/application/service/chat_pipeline/extract_entity_test.go b/internal/application/service/chat_pipeline/extract_entity_test.go
index c58582edd..b7749b5d7 100644
--- a/internal/application/service/chat_pipeline/extract_entity_test.go
+++ b/internal/application/service/chat_pipeline/extract_entity_test.go
@@ -2,8 +2,12 @@ package chatpipeline
import (
"context"
+ "errors"
"strings"
"testing"
+
+ "github.com/Tencent/WeKnora/internal/models/chat"
+ "github.com/Tencent/WeKnora/internal/types"
)
// TestFormater_ParseGraph_FenceVariants exercises the JSON parsing path used
@@ -25,6 +29,10 @@ func TestFormater_ParseGraph_FenceVariants(t *testing.T) {
wantRels int
wantErr bool
errContains string
+ // wantDeclined: ParseGraph fails with ErrModelDeclined (prose, no
+ // JSON). The async chunk task turns that into a skip (#3600); other
+ // callers keep reporting it as a failure.
+ wantDeclined bool
}{
{
name: "wrapped in ```json fence",
@@ -85,6 +93,8 @@ func TestFormater_ParseGraph_FenceVariants(t *testing.T) {
wantRels: 0,
},
{
+ // An empty response is usually transient (gateway hiccup, all
+ // tokens spent on reasoning), so it stays a retriable error.
name: "empty input",
input: "",
wantErr: true,
@@ -97,14 +107,40 @@ func TestFormater_ParseGraph_FenceVariants(t *testing.T) {
errContains: "empty",
},
{
- name: "fenced but body is invalid JSON",
- input: "```json\nnot json at all\n```",
- wantErr: true,
- errContains: "parse",
+ // Fenced body with no JSON structure at all ({ or [) is a prose
+ // refusal dressed in a fence (#3600).
+ name: "fenced but body has no JSON structure",
+ input: "```json\nnot json at all\n```",
+ wantErr: true,
+ wantDeclined: true,
+ },
+ {
+ // Issue #3600: a prose refusal carries no JSON structure at all.
+ name: "prose refusal with no JSON structure is declined",
+ input: "Sorry, I cannot extract a graph from this text.",
+ wantErr: true,
+ wantDeclined: true,
+ },
+ {
+ // Issue #3600: Chinese refusal ("抱歉…") — its UTF-8 bytes show up
+ // in json errors as `invalid character 'æ'`.
+ name: "Chinese prose refusal is declined",
+ input: "抱歉,该文本为目录页,没有可抽取的实体和关系。",
+ wantErr: true,
+ wantDeclined: true,
},
{
- name: "no recoverable JSON, only prose",
- input: "Sorry, I cannot extract a graph from this text.",
+ // A prose fence (e.g. "Step 1" notes) followed by the JSON fence is
+ // not a refusal: the response does contain JSON, so it must stay a
+ // retriable parse failure rather than a terminal skip.
+ name: "prose fence before a JSON fence is not declined",
+ input: "```\nStep 1: list the entities.\n```\n```json\n" + validJSON + "\n```",
+ wantErr: true,
+ },
+ {
+ // Truncated JSON (braces present) remains a real, retriable failure.
+ name: "truncated JSON with structure still fails",
+ input: "[{\"entity\": \"Alice\", \"entity_attrib",
wantErr: true,
errContains: "parse",
},
@@ -123,6 +159,9 @@ func TestFormater_ParseGraph_FenceVariants(t *testing.T) {
if tc.errContains != "" && !strings.Contains(err.Error(), tc.errContains) {
t.Fatalf("error %q does not contain %q", err.Error(), tc.errContains)
}
+ if got := errors.Is(err, ErrModelDeclined); got != tc.wantDeclined {
+ t.Fatalf("errors.Is(err, ErrModelDeclined) = %v, want %v (err=%v)", got, tc.wantDeclined, err)
+ }
return
}
if err != nil {
@@ -266,3 +305,55 @@ func TestNewExtractorUsesGraphOutputBudget(t *testing.T) {
t.Fatalf("MaxTokens = %d, want 8192", got)
}
}
+
+// fixedChat returns one canned response from Chat.
+type fixedChat struct{ resp *types.ChatResponse }
+
+func (c fixedChat) Chat(context.Context, []chat.Message, *chat.ChatOptions) (*types.ChatResponse, error) {
+ return c.resp, nil
+}
+
+func (c fixedChat) ChatStream(
+ context.Context, []chat.Message, *chat.ChatOptions,
+) (<-chan types.StreamResponse, error) {
+ return nil, errors.New("not implemented")
+}
+func (c fixedChat) GetModelName() string { return "fixed" }
+func (c fixedChat) GetModelID() string { return "fixed" }
+
+// A prose answer is only a refusal when the model stopped on its own. If the
+// output budget ran out before any JSON was written (finish_reason=length or
+// any other spelling of it), the stream broke ("incomplete"), or the adapter
+// reported no finish reason, the same prose may be truncation and must stay
+// retriable.
+func TestExtractor_Extract_TruncatedProseIsNotDeclined(t *testing.T) {
+ const prose = "## Step 1: Entity Extraction\n1. Romeo and Juliet is a play"
+ tmpl := &types.PromptTemplateStructured{Description: "extract"}
+
+ cases := []struct {
+ name string
+ finish string
+ wantDeclined bool
+ }{
+ {"model stopped: declined", "stop", true},
+ {"model stopped, upper case: declined", " STOP ", true},
+ {"output truncated: retriable", "length", false},
+ {"anthropic max_tokens: retriable", "max_tokens", false},
+ {"responses max_output_tokens: retriable", "max_output_tokens", false},
+ {"case and space variant: retriable", " Max_Tokens ", false},
+ {"stream broke before stop: retriable", types.FinishReasonIncomplete, false},
+ {"no finish reason reported: retriable", "", false},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ e := NewExtractor(fixedChat{resp: &types.ChatResponse{Content: prose, FinishReason: tc.finish}}, tmpl)
+ graph, err := e.Extract(context.Background(), "chunk text")
+ if err == nil {
+ t.Fatalf("expected error, got graph %+v", graph)
+ }
+ if got := errors.Is(err, ErrModelDeclined); got != tc.wantDeclined {
+ t.Fatalf("errors.Is(err, ErrModelDeclined) = %v, want %v (err=%v)", got, tc.wantDeclined, err)
+ }
+ })
+ }
+}
diff --git a/internal/application/service/extract.go b/internal/application/service/extract.go
index a2e286700..2697fd6b5 100644
--- a/internal/application/service/extract.go
+++ b/internal/application/service/extract.go
@@ -355,6 +355,14 @@ func (s *ChunkExtractService) Handle(ctx context.Context, t *asynq.Task) error {
}
extractor := chatpipeline.NewExtractor(chatModel, template)
graph, err := extractor.Extract(ctx, chunk.Content)
+ if errors.Is(err, chatpipeline.ErrModelDeclined) {
+ // The model answered in prose (e.g. declined a table-of-contents
+ // chunk). Replaying the same input cannot help, so finish the task
+ // instead of letting asynq retry it (issue #3600).
+ logger.Warnf(ctx, "graph extraction skipped for chunk %s: %v", p.ChunkID, err)
+ graphOut["skipped"] = "model_declined"
+ return nil
+ }
if err != nil {
handleErr = err
return err
diff --git a/internal/application/service/extract_graph_declined_test.go b/internal/application/service/extract_graph_declined_test.go
new file mode 100644
index 000000000..1953b8c1f
--- /dev/null
+++ b/internal/application/service/extract_graph_declined_test.go
@@ -0,0 +1,88 @@
+package service
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/models/chat"
+ "github.com/Tencent/WeKnora/internal/types"
+ "github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/hibiken/asynq"
+)
+
+type graphReplyChat struct{ resp *types.ChatResponse }
+
+func (c graphReplyChat) Chat(context.Context, []chat.Message, *chat.ChatOptions) (*types.ChatResponse, error) {
+ return c.resp, nil
+}
+
+func (c graphReplyChat) ChatStream(
+ context.Context, []chat.Message, *chat.ChatOptions,
+) (<-chan types.StreamResponse, error) {
+ return nil, errors.New("not implemented")
+}
+func (c graphReplyChat) GetModelName() string { return "graph-reply" }
+func (c graphReplyChat) GetModelID() string { return "graph-reply" }
+
+type graphChunkRepo struct {
+ interfaces.ChunkRepository
+ chunk *types.Chunk
+}
+
+func (r graphChunkRepo) GetChunkByID(context.Context, uint64, string) (*types.Chunk, error) {
+ return r.chunk, nil
+}
+
+type graphKBRepo struct {
+ interfaces.KnowledgeBaseRepository
+ kb *types.KnowledgeBase
+}
+
+func (r graphKBRepo) GetKnowledgeBaseByID(context.Context, string) (*types.KnowledgeBase, error) {
+ return r.kb, nil
+}
+
+// Issue #3600: a prose refusal finishes the per-chunk task (no asynq retry),
+// while the same prose cut off by the output budget stays a retriable error.
+func TestChunkExtractHandle_ModelDeclinedIsSkipNotRetry(t *testing.T) {
+ const prose = "抱歉,该文本为目录页,没有可抽取的实体和关系。"
+ payload, err := json.Marshal(types.ExtractChunkPayload{TenantID: 1, ChunkID: "c1", ModelID: "m1"})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ cases := []struct {
+ name string
+ finish string
+ wantErr bool
+ }{
+ {"model stopped on its own: skip", "stop", false},
+ {"output truncated: retry", "length", true},
+ {"output truncated (max_tokens): retry", "max_tokens", true},
+ {"stream broke before stop: retry", types.FinishReasonIncomplete, true},
+ {"no finish reason reported: retry", "", true},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ svc := &ChunkExtractService{
+ template: &types.PromptTemplateStructured{Description: "extract"},
+ modelService: &stubModelService{chatModel: graphReplyChat{
+ resp: &types.ChatResponse{Content: prose, FinishReason: tc.finish},
+ }},
+ knowledgeBaseRepo: graphKBRepo{kb: &types.KnowledgeBase{
+ ID: "kb1",
+ ExtractConfig: &types.ExtractConfig{Enabled: true},
+ }},
+ chunkRepo: graphChunkRepo{chunk: &types.Chunk{
+ ID: "c1", KnowledgeBaseID: "kb1", Content: "目录 …… 1",
+ }},
+ }
+ err := svc.Handle(context.Background(), asynq.NewTask(types.TypeChunkExtract, payload))
+ if (err != nil) != tc.wantErr {
+ t.Fatalf("Handle err = %v, wantErr %v", err, tc.wantErr)
+ }
+ })
+ }
+}
diff --git a/internal/handler/initialization_extract_relations_test.go b/internal/handler/initialization_extract_relations_test.go
new file mode 100644
index 000000000..c28e8d178
--- /dev/null
+++ b/internal/handler/initialization_extract_relations_test.go
@@ -0,0 +1,40 @@
+package handler
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/Tencent/WeKnora/internal/config"
+ "github.com/Tencent/WeKnora/internal/models/chat"
+ "github.com/Tencent/WeKnora/internal/types"
+)
+
+type proseReplyChat struct{}
+
+func (proseReplyChat) Chat(context.Context, []chat.Message, *chat.ChatOptions) (*types.ChatResponse, error) {
+ return &types.ChatResponse{Content: "抱歉,这段文本没有可抽取的实体和关系。", FinishReason: "stop"}, nil
+}
+
+func (proseReplyChat) ChatStream(
+ context.Context, []chat.Message, *chat.ChatOptions,
+) (<-chan types.StreamResponse, error) {
+ return nil, errors.New("not implemented")
+}
+func (proseReplyChat) GetModelName() string { return "prose" }
+func (proseReplyChat) GetModelID() string { return "prose" }
+
+// The graph-settings "try extract" button must report a prose refusal as a
+// failure. Returning an empty graph would make the UI clear the user's
+// few-shot nodes/relations and show "success" (#3606 review).
+func TestExtractRelationsFromTextReportsModelRefusal(t *testing.T) {
+ h := &InitializationHandler{config: &config.Config{
+ ExtractManager: &config.ExtractManagerConfig{
+ ExtractGraph: &types.PromptTemplateStructured{Description: "extract"},
+ },
+ }}
+ res, err := h.extractRelationsFromText(context.Background(), "目录", nil, proseReplyChat{})
+ if err == nil {
+ t.Fatalf("expected an error for a prose refusal, got %+v", res)
+ }
+}
diff --git a/internal/models/chat/ollama.go b/internal/models/chat/ollama.go
index 7597863e8..f04f30d0a 100644
--- a/internal/models/chat/ollama.go
+++ b/internal/models/chat/ollama.go
@@ -164,6 +164,7 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
var responseContent string
var toolCalls []types.LLMToolCall
var promptTokens, completionTokens int
+ var finishReason string
// 使用 Ollama 客户端发送请求
err := c.ollamaService.Chat(ctx, chatReq, func(resp ollamaapi.ChatResponse) error {
@@ -173,6 +174,11 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
responseContent = resp.Message.Thinking
}
toolCalls = c.toolCallTo(resp.Message.ToolCalls)
+ // done_reason 是 "stop" / "length" 等;不透传的话调用方分不清拒答和
+ // num_predict 打满导致的截断(图谱抽取据此决定跳过还是重试)。
+ if resp.DoneReason != "" {
+ finishReason = resp.DoneReason
+ }
// 获取token计数。eval_count 本身就是回答的 token 数,不含 prompt
// (https://github.com/ollama/ollama/blob/main/docs/api.md),所以不能再
@@ -197,9 +203,10 @@ func (c *OllamaChat) Chat(ctx context.Context, messages []Message, opts *ChatOpt
api.LogUsage(ctx, c.modelName, &usage)
return &types.ChatResponse{
- Content: responseContent,
- ToolCalls: toolCalls,
- Usage: usage,
+ Content: responseContent,
+ ToolCalls: toolCalls,
+ FinishReason: finishReason,
+ Usage: usage,
}, nil
}
diff --git a/internal/models/chat/ollama_test.go b/internal/models/chat/ollama_test.go
index 837702699..0a0254cf4 100644
--- a/internal/models/chat/ollama_test.go
+++ b/internal/models/chat/ollama_test.go
@@ -1,10 +1,13 @@
package chat
import (
+ "context"
+ "fmt"
"net/http"
"net/http/httptest"
"testing"
+ "github.com/Tencent/WeKnora/internal/models/utils/ollama"
secutils "github.com/Tencent/WeKnora/internal/utils"
)
@@ -32,3 +35,42 @@ func TestResolveImageForOllamaBlocksRedirectToInternalURL(t *testing.T) {
t.Fatalf("resolveImageForOllama returned data after redirect to blocked internal URL")
}
}
+
+// Graph extraction tells a prose refusal from an answer cut off by num_predict
+// through FinishReason, so the non-streaming Chat must carry Ollama's
+// done_reason instead of leaving it empty.
+func TestOllamaChatPropagatesDoneReason(t *testing.T) {
+ for _, reason := range []string{"stop", "length"} {
+ t.Run(reason, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ switch r.URL.Path {
+ case "/":
+ w.WriteHeader(http.StatusOK)
+ case "/api/tags":
+ _, _ = fmt.Fprint(w, `{"models":[{"name":"m:latest"}]}`)
+ case "/api/chat":
+ _, _ = fmt.Fprintf(w, `{"model":"m","message":{"role":"assistant","content":"Step 1"},`+
+ `"done":true,"done_reason":%q,"eval_count":3}`, reason)
+ default:
+ http.NotFound(w, r)
+ }
+ }))
+ defer server.Close()
+ t.Setenv("OLLAMA_BASE_URL", server.URL)
+ t.Setenv("OLLAMA_OPTIONAL", "")
+ svc, err := ollama.GetOllamaService()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ c := &OllamaChat{modelName: "m", ollamaService: svc}
+ resp, err := c.Chat(context.Background(), []Message{{Role: "user", Content: "hi"}}, &ChatOptions{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if resp.FinishReason != reason {
+ t.Fatalf("FinishReason = %q, want %q", resp.FinishReason, reason)
+ }
+ })
+ }
+}