Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(summary): 坏 JSON 不再冒充文档摘要(截断/畸形 JSON 走失败路径,纯文本兜底保留) (#3769)

linus-liu-weba15dfdd94149
diff --git a/internal/application/service/extract_data_table_summary_test.go b/internal/application/service/extract_data_table_summary_test.go
index d94338131..1079c910f 100644
--- a/internal/application/service/extract_data_table_summary_test.go
+++ b/internal/application/service/extract_data_table_summary_test.go
@@ -20,6 +20,46 @@ import (
"github.com/stretchr/testify/require"
)
+// The data-table path reads response.Content directly and never applies the
+// document-summary JSON contract. A table or column description cut off at its
+// own budget (512 / 2048 tokens) is still worth indexing, so these callers must
+// keep returning it instead of failing the whole table summary task.
+func TestTableDescriptionCallersKeepTruncatedReplies(t *testing.T) {
+ truncated := &types.ChatResponse{
+ Content: "| country | capital |\n| --- | --- |\n| Alpha Republic | North",
+ FinishReason: "length",
+ }
+ service := &DataTableSummaryService{}
+
+ t.Run("table description", func(t *testing.T) {
+ got, err := service.generateTableDescription(
+ context.Background(),
+ &fixedResponseSummaryChat{response: truncated},
+ "data_analysis_table", "country TEXT, capital TEXT", "alpha / north city", "",
+ )
+ if err != nil {
+ t.Fatalf("generateTableDescription() error = %v", err)
+ }
+ if !strings.Contains(got, truncated.Content) {
+ t.Fatalf("truncated table description was dropped:\n%s", got)
+ }
+ })
+
+ t.Run("column descriptions", func(t *testing.T) {
+ got, err := service.generateColumnDescriptions(
+ context.Background(),
+ &fixedResponseSummaryChat{response: truncated},
+ "data_analysis_table", "country TEXT, capital TEXT", "alpha / north city", "",
+ )
+ if err != nil {
+ t.Fatalf("generateColumnDescriptions() error = %v", err)
+ }
+ if !strings.Contains(got, truncated.Content) {
+ t.Fatalf("truncated column descriptions were dropped:\n%s", got)
+ }
+ })
+}
+
func TestBuildSampleDataDescriptionIncludesDataAnalysisRows(t *testing.T) {
service := &DataTableSummaryService{}
result := &types.ToolResult{Data: map[string]interface{}{
diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go
index 07e49da3b..5150d015f 100644
--- a/internal/application/service/knowledge_process.go
+++ b/internal/application/service/knowledge_process.go
@@ -12,6 +12,7 @@ import (
"github.com/Tencent/WeKnora/internal/application/access"
"github.com/Tencent/WeKnora/internal/application/repository"
+ chatpipeline "github.com/Tencent/WeKnora/internal/application/service/chat_pipeline"
"github.com/Tencent/WeKnora/internal/application/service/retriever"
"github.com/Tencent/WeKnora/internal/common"
werrors "github.com/Tencent/WeKnora/internal/errors"
@@ -864,15 +865,81 @@ type documentProfileOutput struct {
TypicalQuestion string `json:"typical_question"`
}
+// normalizeSummaryReplyContent strips a UTF-8 BOM, the surrounding whitespace
+// and one markdown code fence from a summary reply, so shape detection and
+// parsing look at the same text. A fence without a newline carries no payload
+// we can classify, so it normalizes to the empty string.
+func normalizeSummaryReplyContent(content string) string {
+ content = strings.TrimSpace(strings.TrimPrefix(content, "\uFEFF"))
+ if rest, ok := strings.CutPrefix(content, "```"); ok {
+ // Drop the fence's info string (```json) and the newline after it.
+ idx := strings.IndexByte(rest, '\n')
+ if idx < 0 {
+ return ""
+ }
+ content = strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(rest[idx+1:]), "```"))
+ }
+ return content
+}
+
+// summaryReplyIsJSONShaped reports whether the reply really is a structured
+// JSON payload, either directly or inside a markdown code fence. Only such
+// replies are held to the structured contract below; anything else is a
+// legacy plain-text summary.
+//
+// An object opening ("{") always counts, parseable or not: half an object is
+// exactly the reply that must never be stored as a summary. A bracket opening
+// is ambiguous — "[1] …" and "[文档摘要] …" are prose with a leading citation,
+// not broken JSON — so it only counts when the reply is a JSON array or opens
+// an array of objects/strings.
+func summaryReplyIsJSONShaped(content string) bool {
+ content = normalizeSummaryReplyContent(content)
+ if content == "" {
+ return false
+ }
+ switch content[0] {
+ case '{':
+ return true
+ case '[':
+ var arr []json.RawMessage
+ if json.Unmarshal([]byte(content), &arr) == nil {
+ return true
+ }
+ first := strings.TrimLeft(content[1:], " \t\r\n")
+ return strings.HasPrefix(first, "{") ||
+ strings.HasPrefix(first, "[") ||
+ strings.HasPrefix(first, `"`)
+ default:
+ return false
+ }
+}
+
// parseDocumentSummaryOutput accepts both the structured JSON reply and a
// legacy plain-text summary (custom templates, older models). Plain text is
// stored as the summary with no profile, so nothing that worked before
// regresses; only the knowledge-base aggregation loses that document's topics.
-func parseDocumentSummaryOutput(content string) *documentSummaryResult {
- content = strings.TrimSpace(content)
+//
+// A reply that opens like JSON is held to the JSON contract instead: a broken
+// object (output cut off at MaxTokens, an unescaped newline inside a string)
+// or an object with no usable text returns errSummaryOutputNotParsable. The
+// raw reply used to be stored as the summary, which put half a JSON object in
+// knowledges.description, marked the summary completed so Asynq never retried
+// it, and embedded the same fragment into the RAG index.
+func parseDocumentSummaryOutput(content string) (*documentSummaryResult, error) {
+ // Some providers prefix the reply with a UTF-8 BOM; TrimSpace alone would
+ // leave it in place, hide the "{" and let half an object through as prose.
+ content = strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(content), "\uFEFF"))
+ if content == "" {
+ return nil, errEmptySummaryOutput
+ }
+ jsonShaped := summaryReplyIsJSONShaped(content)
var out documentProfileOutput
if err := common.ParseLLMJsonResponse(content, &out); err != nil {
- return &documentSummaryResult{Summary: content}
+ if jsonShaped {
+ return nil, fmt.Errorf("%w: %w", errSummaryOutputNotParsable, err)
+ }
+ // Legacy plain text: keep it as the summary with no profile.
+ return &documentSummaryResult{Summary: content}, nil
}
summary := strings.TrimSpace(out.Summary)
profile := (&types.KnowledgeProfile{
@@ -885,11 +952,15 @@ func parseDocumentSummaryOutput(content string) *documentSummaryResult {
summary = profile.Gist
}
if summary == "" {
- // JSON without usable text: fall back to the raw content so the
- // caller's empty-output handling still applies.
- return &documentSummaryResult{Summary: content}
+ if jsonShaped {
+ // JSON without usable text is as unusable as broken JSON: the
+ // caller must fail the attempt instead of storing the object.
+ return nil, errSummaryOutputNotParsable
+ }
+ // Plain text that happened to contain balanced JSON punctuation.
+ return &documentSummaryResult{Summary: content}, nil
}
- return &documentSummaryResult{Summary: summary, Profile: profile}
+ return &documentSummaryResult{Summary: summary, Profile: profile}, nil
}
// buildSummaryChunkContent is the text embedded for the document-level
@@ -921,13 +992,23 @@ const imageDominatedTextThreshold = 200
var (
errInsufficientSummaryContent = errors.New("insufficient text content for summary generation")
errEmptySummaryOutput = errors.New("summary model returned empty output")
+ // errSummaryOutputNotParsable marks a reply that is JSON-shaped but yields
+ // no usable summary text — broken JSON (cut off at the completion budget,
+ // unescaped newline) or an object without summary/gist. It is deliberately
+ // not stored: the raw reply is not a summary.
+ errSummaryOutputNotParsable = errors.New("summary model returned JSON without a usable summary")
+ // errSummaryOutputTruncated marks a reply the provider stopped at the
+ // completion budget, so its JSON can never be trusted as complete.
+ errSummaryOutputTruncated = errors.New("summary model output was truncated at the completion budget")
)
const summaryFallbackMaxRunes = 500
// validateSummaryOutput rejects successful model responses that contain no
-// user-visible text. Treating whitespace-only output as an error lets Asynq
-// retry the summary task instead of persisting description="" as completed.
+// user-visible text. It only looks at the content shape: a provider that
+// stopped at the completion budget can still return usable text, and the older
+// plain-text consumers (custom summary templates, table/column descriptions)
+// deliberately keep whatever the model produced.
func validateSummaryOutput(response *types.ChatResponse) (string, error) {
if response == nil {
return "", errEmptySummaryOutput
@@ -939,6 +1020,27 @@ func validateSummaryOutput(response *types.ChatResponse) (string, error) {
return content, nil
}
+// validateStructuredSummaryDocument is the validator for the document-summary
+// entry point (getSummary). On top of validateSummaryOutput it rejects a reply
+// the provider cut off at the completion budget — but only when the reply was
+// producing a structured summary, because that is the case where the truncated
+// text is half a JSON object. A plain-text summary that hit the budget is
+// still usable text and keeps the legacy fallback.
+//
+// The shape is judged before the finish reason, so "finish_reason=length" can
+// never turn a plain-text reply into a failure on its own.
+func validateStructuredSummaryDocument(response *types.ChatResponse) (string, error) {
+ content, err := validateSummaryOutput(response)
+ if err != nil {
+ return "", err
+ }
+ structured := summaryReplyIsJSONShaped(content)
+ if structured && chatpipeline.IsLengthFinishReason(response.FinishReason) {
+ return "", errSummaryOutputTruncated
+ }
+ return content, nil
+}
+
// firstTextChunkSummaryFallback preserves the existing deterministic fallback:
// use the first already-ordered text chunk and cap it by runes so Chinese and
// emoji are never cut in the middle of a UTF-8 sequence.
@@ -1173,12 +1275,16 @@ func (s *knowledgeService) getSummary(ctx context.Context,
logger.GetLogger(ctx).WithField("error", err).Errorf("GetSummary failed")
return nil, err
}
- content, err := validateSummaryOutput(summary)
+ content, err := validateStructuredSummaryDocument(summary)
if err != nil {
logger.GetLogger(ctx).WithField("error", err).Warnf("GetSummary returned no usable content")
return nil, err
}
- result := parseDocumentSummaryOutput(content)
+ result, err := parseDocumentSummaryOutput(content)
+ if err != nil {
+ logger.GetLogger(ctx).WithField("error", err).Warnf("GetSummary returned unusable content")
+ return nil, err
+ }
logger.GetLogger(ctx).WithField("summary", result.Summary).
WithField("has_profile", result.Profile != nil).Infof("GetSummary success")
return result, nil
diff --git a/internal/application/service/knowledge_summary_reconstruction_test.go b/internal/application/service/knowledge_summary_reconstruction_test.go
index b77232de4..f6ea3d1c0 100644
--- a/internal/application/service/knowledge_summary_reconstruction_test.go
+++ b/internal/application/service/knowledge_summary_reconstruction_test.go
@@ -8,6 +8,7 @@ import (
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/types"
"github.com/Tencent/WeKnora/internal/types/interfaces"
+ "github.com/stretchr/testify/require"
)
type summaryContentCaptureChat struct {
@@ -81,3 +82,133 @@ func TestGetSummaryReconstructsTableChunksWithSyntheticHeaders(t *testing.T) {
t.Fatalf("summary content mismatch:\n got: %q\nwant: %q", got, want)
}
}
+
+// fixedResponseSummaryChat replays one provider response so a test can drive
+// getSummary with the replies that used to be persisted verbatim.
+type fixedResponseSummaryChat struct {
+ response *types.ChatResponse
+}
+
+func (m *fixedResponseSummaryChat) Chat(
+ _ context.Context,
+ _ []chat.Message,
+ _ *chat.ChatOptions,
+) (*types.ChatResponse, error) {
+ return m.response, nil
+}
+
+func (m *fixedResponseSummaryChat) ChatStream(
+ context.Context,
+ []chat.Message,
+ *chat.ChatOptions,
+) (<-chan types.StreamResponse, error) {
+ return nil, nil
+}
+
+func (m *fixedResponseSummaryChat) GetModelName() string { return "summary-fixed" }
+func (m *fixedResponseSummaryChat) GetModelID() string { return "summary-fixed" }
+
+// getSummary is the single call site both summary entry points share
+// (ProcessSummaryGeneration and RegenerateKnowledgeSummary), so the error
+// returned here is what routes an unusable reply into their existing
+// retry/failed path instead of knowledge.description + SummaryStatusCompleted.
+func TestGetSummaryRejectsUnusableModelOutput(t *testing.T) {
+ tests := []struct {
+ name string
+ response *types.ChatResponse
+ wantErr error
+ }{
+ {
+ name: "reply truncated at the completion budget",
+ response: &types.ChatResponse{
+ Content: `{"summary": "cut off mid-`, FinishReason: "length",
+ },
+ wantErr: errSummaryOutputTruncated,
+ },
+ {
+ name: "JSON-shaped reply that does not parse",
+ response: &types.ChatResponse{
+ Content: "{\"summary\": \"line one\nline two\"}",
+ },
+ wantErr: errSummaryOutputNotParsable,
+ },
+ {
+ name: "empty reply",
+ response: &types.ChatResponse{Content: " "},
+ wantErr: errEmptySummaryOutput,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ service := &knowledgeService{
+ config: &config.Config{Conversation: &config.ConversationConfig{
+ GenerateSummaryPrompt: "Summarize the document.",
+ }},
+ chunkRepo: summaryImageInfoChunkRepo{},
+ }
+ body := "The leave policy grants twenty days of annual leave to every employee."
+ result, err := service.getSummary(
+ context.Background(),
+ &fixedResponseSummaryChat{response: tt.response},
+ &types.Knowledge{ID: "knowledge-1"},
+ []*types.Chunk{{
+ ID: "first", Content: body, ChunkIndex: 0,
+ StartAt: 0, EndAt: len([]rune(body)),
+ }},
+ )
+ require.ErrorIs(t, err, tt.wantErr)
+ require.Nil(t, result, "an unusable reply must not become a summary result")
+ })
+ }
+}
+
+// A plain-text template that hits the completion budget still returned usable
+// text, and only a structured reply can be half a JSON object. Treating the
+// finish reason alone as failure turned a truncated-but-usable custom-template
+// summary into a retry loop ending in SummaryStatusFailed.
+func TestGetSummaryKeepsTruncatedPlainTextReply(t *testing.T) {
+ tests := []struct {
+ name string
+ content string
+ }{
+ {
+ name: "plain text at the budget",
+ content: "This document explains the leave policy and the twenty days",
+ },
+ {
+ name: "bracketed prose at the budget",
+ content: "[文档摘要] 本文件说明请假制度与年度额度",
+ },
+ {
+ name: "numbered prose at the budget",
+ content: "[1] 本文件说明请假制度与年度额度",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ service := &knowledgeService{
+ config: &config.Config{Conversation: &config.ConversationConfig{
+ GenerateSummaryPrompt: "Summarize the document.",
+ }},
+ chunkRepo: summaryImageInfoChunkRepo{},
+ }
+ body := "The leave policy grants twenty days of annual leave to every employee."
+ result, err := service.getSummary(
+ context.Background(),
+ &fixedResponseSummaryChat{response: &types.ChatResponse{
+ Content: tt.content, FinishReason: "length",
+ }},
+ &types.Knowledge{ID: "knowledge-1"},
+ []*types.Chunk{{
+ ID: "first", Content: body, ChunkIndex: 0,
+ StartAt: 0, EndAt: len([]rune(body)),
+ }},
+ )
+ require.NoError(t, err)
+ require.NotNil(t, result)
+ require.Equal(t, tt.content, result.Summary)
+ })
+ }
+}
diff --git a/internal/application/service/knowledge_summary_test.go b/internal/application/service/knowledge_summary_test.go
index 69fb5927f..2eca03bf2 100644
--- a/internal/application/service/knowledge_summary_test.go
+++ b/internal/application/service/knowledge_summary_test.go
@@ -107,6 +107,10 @@ func TestCheckSufficientSummaryContent_ThresholdOverride(t *testing.T) {
}
}
+// validateSummaryOutput is the content-only contract shared by every caller
+// that just wants "did the model return text". A provider that stopped at the
+// completion budget still returned text, so the finish reason alone must not
+// turn it into an error here.
func TestValidateSummaryOutput(t *testing.T) {
tests := []struct {
name string
@@ -126,6 +130,23 @@ func TestValidateSummaryOutput(t *testing.T) {
response: &types.ChatResponse{Content: " useful summary \n"},
want: "useful summary",
},
+ {
+ name: "a normal stop is accepted",
+ response: &types.ChatResponse{Content: "useful summary", FinishReason: "stop"},
+ want: "useful summary",
+ },
+ {
+ // The table/column description callers rely on this: a description
+ // cut off at 512/2048 tokens is still worth indexing.
+ name: "plain text at the budget is accepted",
+ response: &types.ChatResponse{Content: "partial", FinishReason: "max_tokens"},
+ want: "partial",
+ },
+ {
+ name: "JSON at the budget is accepted here, rejected by the caller",
+ response: &types.ChatResponse{Content: `{"summary": "half a doc`, FinishReason: "length"},
+ want: `{"summary": "half a doc`,
+ },
}
for _, tt := range tests {
@@ -133,7 +154,138 @@ func TestValidateSummaryOutput(t *testing.T) {
got, err := validateSummaryOutput(tt.response)
if tt.wantError {
if !errors.Is(err, errEmptySummaryOutput) {
- t.Fatalf("expected errEmptySummaryOutput, got %v", err)
+ t.Fatalf("expected %v, got %v", errEmptySummaryOutput, err)
+ }
+ return
+ }
+ if err != nil {
+ t.Fatalf("unexpected error: %v", err)
+ }
+ if got != tt.want {
+ t.Fatalf("summary = %q, want %q", got, tt.want)
+ }
+ })
+ }
+}
+
+// validateStructuredSummaryDocument is the document-summary gate: it only adds
+// the truncation verdict for replies that are actually producing structured
+// JSON, so a plain-text template that hits the budget keeps the legacy
+// fallback instead of failing the task.
+func TestValidateStructuredSummaryDocument(t *testing.T) {
+ tests := []struct {
+ name string
+ response *types.ChatResponse
+ want string
+ wantError bool
+ wantErr error // checked when set; otherwise wantError means errEmptySummaryOutput
+ }{
+ {name: "nil response rejected", response: nil, wantError: true},
+ {name: "empty response rejected", response: &types.ChatResponse{}, wantError: true},
+ {
+ name: "plain text at the budget is not an error",
+ response: &types.ChatResponse{
+ Content: "This document explains the leave policy and the twenty days",
+ FinishReason: "length",
+ },
+ want: "This document explains the leave policy and the twenty days",
+ },
+ {
+ name: "plain text with max_tokens is not an error",
+ response: &types.ChatResponse{
+ Content: "partial prose summary",
+ FinishReason: "max_tokens",
+ },
+ want: "partial prose summary",
+ },
+ {
+ name: "bracketed prose at the budget is not an error",
+ response: &types.ChatResponse{
+ Content: "[文档摘要] 本文件说明请假制度与年度额度",
+ FinishReason: "length",
+ },
+ want: "[文档摘要] 本文件说明请假制度与年度额度",
+ },
+ {
+ name: "numbered prose at the budget is not an error",
+ response: &types.ChatResponse{
+ Content: "[1] 本文件说明请假制度与年度额度",
+ FinishReason: "length",
+ },
+ want: "[1] 本文件说明请假制度与年度额度",
+ },
+ {
+ name: "truncated JSON object is rejected",
+ response: &types.ChatResponse{
+ Content: `{"summary": "half a doc`,
+ FinishReason: "length",
+ },
+ wantError: true,
+ wantErr: errSummaryOutputTruncated,
+ },
+ {
+ name: "complete but malformed JSON at the budget is rejected",
+ response: &types.ChatResponse{
+ Content: "{\"summary\": \"line one\nline two\"}",
+ FinishReason: "length",
+ },
+ wantError: true,
+ wantErr: errSummaryOutputTruncated,
+ },
+ {
+ name: "truncated fenced JSON is rejected",
+ response: &types.ChatResponse{
+ Content: "```json\n{\"summary\": \"half a doc",
+ FinishReason: "length",
+ },
+ wantError: true,
+ wantErr: errSummaryOutputTruncated,
+ },
+ {
+ name: "BOM-prefixed truncated JSON is rejected",
+ response: &types.ChatResponse{
+ Content: "\uFEFF{\"summary\": \"half a doc",
+ FinishReason: "length",
+ },
+ wantError: true,
+ wantErr: errSummaryOutputTruncated,
+ },
+ {
+ name: "truncated JSON array of objects is rejected",
+ response: &types.ChatResponse{
+ Content: `[{"summary": "half a doc`,
+ FinishReason: "length",
+ },
+ wantError: true,
+ wantErr: errSummaryOutputTruncated,
+ },
+ {
+ name: "malformed JSON without a length stop is left to the parser",
+ response: &types.ChatResponse{
+ Content: "{\"summary\": \"line one\nline two\"}",
+ },
+ want: "{\"summary\": \"line one\nline two\"}",
+ },
+ {
+ name: "structured JSON with a normal stop is accepted",
+ response: &types.ChatResponse{
+ Content: `{"summary": "Two sentences."}`,
+ FinishReason: "stop",
+ },
+ want: `{"summary": "Two sentences."}`,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := validateStructuredSummaryDocument(tt.response)
+ if tt.wantError {
+ wantErr := tt.wantErr
+ if wantErr == nil {
+ wantErr = errEmptySummaryOutput
+ }
+ if !errors.Is(err, wantErr) {
+ t.Fatalf("expected %v, got %v", wantErr, err)
}
return
}
diff --git a/internal/application/service/knowledgebase_profile_test.go b/internal/application/service/knowledgebase_profile_test.go
index 4ac525b1f..586859868 100644
--- a/internal/application/service/knowledgebase_profile_test.go
+++ b/internal/application/service/knowledgebase_profile_test.go
@@ -358,33 +358,132 @@ func TestKnowledgeBaseProfileHandle(t *testing.T) {
}
func TestParseDocumentSummaryOutput(t *testing.T) {
- t.Run("structured JSON", func(t *testing.T) {
- out := parseDocumentSummaryOutput("```json\n" +
- `{"summary":"Two sentences.","gist":"A gist","topics":["a","b"],` +
- `"doc_type":"policy","typical_question":"Why?"}` +
- "\n```")
- assert.Equal(t, "Two sentences.", out.Summary)
- require.NotNil(t, out.Profile)
- assert.Equal(t, "A gist", out.Profile.Gist)
- assert.Equal(t, []string{"a", "b"}, out.Profile.Topics)
- assert.Equal(t, "policy", out.Profile.DocType)
- })
- t.Run("plain text keeps working", func(t *testing.T) {
- out := parseDocumentSummaryOutput(" This document explains leave policy. ")
- assert.Equal(t, "This document explains leave policy.", out.Summary)
- assert.Nil(t, out.Profile)
- })
- t.Run("empty content sentinel", func(t *testing.T) {
- out := parseDocumentSummaryOutput(`{"summary": "No textual content was extractable from this document.",` +
- ` "gist": "", "topics": [], "doc_type": "", "typical_question": ""}`)
- assert.Equal(t, "No textual content was extractable from this document.", out.Summary)
- assert.Nil(t, out.Profile)
- })
- t.Run("gist fills a missing summary", func(t *testing.T) {
- out := parseDocumentSummaryOutput(`{"gist":"Only a gist","topics":["x"]}`)
- assert.Equal(t, "Only a gist", out.Summary)
- require.NotNil(t, out.Profile)
- })
+ fullJSON := `{"summary":"Two sentences.","gist":"A gist","topics":["a","b"],` +
+ `"doc_type":"policy","typical_question":"Why?"}`
+ fullProfile := &types.KnowledgeProfile{
+ Gist: "A gist",
+ Topics: []string{"a", "b"},
+ DocType: "policy",
+ TypicalQuestion: "Why?",
+ }
+ // The reported failure shape: a 209-rune reply cut off at MaxTokens in the
+ // middle of a string. It used to be stored raw as knowledge.Description,
+ // marked completed (so Asynq never retried) and embedded as the summary chunk.
+ truncatedJSON := `{"summary": "` + strings.Repeat("截", 130) + `", "gist": "` + strings.Repeat("断", 54)
+
+ tests := []struct {
+ name string
+ content string
+ want string
+ profile *types.KnowledgeProfile
+ wantErr error
+ }{
+ {
+ name: "structured JSON keeps summary and profile",
+ content: fullJSON,
+ want: "Two sentences.",
+ profile: fullProfile,
+ },
+ {
+ name: "fenced JSON keeps summary and profile",
+ content: "```json\n" + fullJSON + "\n```",
+ want: "Two sentences.",
+ profile: fullProfile,
+ },
+ {
+ name: "gist fills a missing summary",
+ content: `{"gist":"Only a gist","topics":["x"]}`,
+ want: "Only a gist",
+ profile: &types.KnowledgeProfile{Gist: "Only a gist", Topics: []string{"x"}},
+ },
+ {
+ // The template's refusal reply is valid JSON with usable text, so
+ // it keeps working (it carries no profile fields).
+ name: "JSON refusal with an empty profile is kept",
+ content: `{"summary": "No textual content was extractable from this document.",` +
+ ` "gist": "", "topics": [], "doc_type": "", "typical_question": ""}`,
+ want: "No textual content was extractable from this document.",
+ },
+ {
+ name: "truncated JSON is rejected, never stored raw",
+ content: truncatedJSON,
+ wantErr: errSummaryOutputNotParsable,
+ },
+ {
+ // A UTF-8 BOM is not whitespace: before the shape check stripped
+ // it, the leading "{" stayed hidden and this half object was
+ // handed back as the summary.
+ name: "BOM-prefixed truncated JSON is rejected",
+ content: "\uFEFF" + truncatedJSON,
+ wantErr: errSummaryOutputNotParsable,
+ },
+ {
+ name: "BOM-prefixed structured JSON still parses",
+ content: "\uFEFF" + fullJSON,
+ want: "Two sentences.",
+ profile: fullProfile,
+ },
+ {
+ // Prose that opens with a bracketed citation is not broken JSON.
+ name: "bracketed prose keeps the legacy fallback",
+ content: "[文档摘要] 本文件说明请假制度与年度额度。",
+ want: "[文档摘要] 本文件说明请假制度与年度额度。",
+ },
+ {
+ name: "numbered prose keeps the legacy fallback",
+ content: "[1] 本文件说明请假制度与年度额度。",
+ want: "[1] 本文件说明请假制度与年度额度。",
+ },
+ {
+ // A real JSON array is structured output that does not satisfy the
+ // object contract, so it is rejected rather than stored raw.
+ name: "JSON array without the object contract is rejected",
+ content: `["a","b"]`,
+ wantErr: errSummaryOutputNotParsable,
+ },
+ {
+ name: "truncated JSON array is rejected",
+ content: `[{"summary": "half a doc`,
+ wantErr: errSummaryOutputNotParsable,
+ },
+ {
+ name: "JSON with an unescaped newline is rejected",
+ content: "{\"summary\": \"First line\nSecond line\", \"gist\": \"g\"}",
+ wantErr: errSummaryOutputNotParsable,
+ },
+ {
+ name: "JSON without summary or gist is rejected",
+ content: `{"topics":["a"],"doc_type":"policy"}`,
+ wantErr: errSummaryOutputNotParsable,
+ },
+ {
+ name: "whitespace-only reply is rejected",
+ content: " \n\t ",
+ wantErr: errEmptySummaryOutput,
+ },
+ {
+ // Custom templates and older models answer in prose; that fallback
+ // is deliberate and must survive.
+ name: "plain text keeps the legacy fallback",
+ content: " This document explains leave policy. ",
+ want: "This document explains leave policy.",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ out, err := parseDocumentSummaryOutput(tt.content)
+ if tt.wantErr != nil {
+ require.ErrorIs(t, err, tt.wantErr)
+ require.Nil(t, out, "a rejected reply must never be handed back as a summary")
+ return
+ }
+ require.NoError(t, err)
+ require.NotNil(t, out)
+ assert.Equal(t, tt.want, out.Summary)
+ assert.Equal(t, tt.profile, out.Profile)
+ })
+ }
}
func TestBuildSummaryChunkContent(t *testing.T) {