RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(memory): align extracted message document id with message_id (#19326)
jay77721545753e248f8
diff --git a/internal/service/memory_extractor.go b/internal/service/memory_extractor.goindex 67273cbac5..fc63fa879d 100644--- a/internal/service/memory_extractor.go+++ b/internal/service/memory_extractor.go@@ -36,7 +36,6 @@ package serviceimport ("context"- "crypto/sha256""encoding/json""errors""fmt"@@ -247,30 +246,20 @@ func (s *MemoryMessageService) extractByLLM(ctx context.Context, mem *CreateMemo// logical message fields are set here; the doc engine maps them to// storage fields (including tokenization) at insert time.func buildExtractedMessage(messageID, sourceID int64, memoryID string, msg MemoryMessage, item extractedMemory, now time.Time) map[string]any {- validAt, hasExplicitValidAt := normalizeMemoryTime(item.ValidAt)- if !hasExplicitValidAt {+ validAt, ok := normalizeMemoryTime(item.ValidAt)+ if !ok {validAt = now.Format(memoryTimeLayout)}- invalidAt, hasExplicitInvalidAt := normalizeMemoryTime(item.InvalidAt)- if strings.TrimSpace(item.InvalidAt) != "" {- if !hasExplicitInvalidAt {- invalidAt = now.Format(memoryTimeLayout)- }+ invalidAt, ok := normalizeMemoryTime(item.InvalidAt)+ if strings.TrimSpace(item.InvalidAt) != "" && !ok {+ invalidAt = now.Format(memoryTimeLayout)}var storedInvalidAt anyif invalidAt != "" {storedInvalidAt = invalidAt}- fingerprintValidAt := ""- if hasExplicitValidAt {- fingerprintValidAt = validAt- }- fingerprintInvalidAt := ""- if hasExplicitInvalidAt {- fingerprintInvalidAt = invalidAt- }return map[string]any{- "id": extractedMessageDocumentID(memoryID, sourceID, item.MessageType, item.Content, fingerprintValidAt, fingerprintInvalidAt),+ "id": fmt.Sprintf("%s_%d", memoryID, messageID),"message_id": messageID,"message_type": item.MessageType,"source_id": sourceID,@@ -286,17 +275,6 @@ func buildExtractedMessage(messageID, sourceID int64, memoryID string, msg Memor}}-// extractedMessageDocumentID returns the chunk-store id for an extracted item.-// sourceID identifies the immutable raw message, while the normalized content-// fingerprint distinguishes multiple extracts from that source. Only explicit,-// parseable timestamps participate, so a runtime fallback cannot change the-// identity on retry. Repeated task executions therefore upsert the same chunk-// instead of creating duplicates.-func extractedMessageDocumentID(memoryID string, sourceID int64, messageType, content, validAt, invalidAt string) string {- fingerprint := sha256.Sum256([]byte(strings.Join([]string{messageType, content, validAt, invalidAt}, "\x00")))- return fmt.Sprintf("%s_%d_%x", memoryID, sourceID, fingerprint[:8])-}-// parseMemoryExtraction ports memory.utils.msg_util.get_json_result_from_llm_response// plus the per-type flattening in extract_by_llm. Only the configured// extract types are collected; unparseable responses yield an empty list.diff --git a/internal/service/memory_extractor_test.go b/internal/service/memory_extractor_test.goindex 8357b10774..fd305ffc1e 100644--- a/internal/service/memory_extractor_test.go+++ b/internal/service/memory_extractor_test.go@@ -99,45 +99,20 @@ func TestBuildExtractedMessageValidAtSemantics(t *testing.T) {}}-// TestBuildExtractedMessageUsesStableDocumentIDAcrossRetries ensures fallback-// timestamps do not become part of the chunk-store identity. A retry may run-// at a different time and allocate a new logical message id, but it must still-// overwrite the same extracted document.-func TestBuildExtractedMessageUsesStableDocumentIDAcrossRetries(t *testing.T) {+// TestBuildExtractedMessageUsesStandardDocumentID ensures extracted items+// use the standard memoryID_messageID identity expected by GetMessageContent.+func TestBuildExtractedMessageUsesStandardDocumentID(t *testing.T) {msg := MemoryMessage{UserID: "u1", AgentID: "a1", SessionID: "s1"}- firstAttempt := time.Date(2026, 8, 20, 10, 5, 0, 0, time.Local)- retryAttempt := firstAttempt.Add(10 * time.Second)-- for _, test := range []struct {- name string- item extractedMemory- }{- {- name: "omitted valid at",- item: extractedMemory{MessageType: "fact", Content: "likes coffee"},- },- {- name: "invalid timestamps",- item: extractedMemory{- MessageType: "fact",- Content: "likes coffee",- ValidAt: "not a timestamp",- InvalidAt: "also not a timestamp",- },- },- } {- t.Run(test.name, func(t *testing.T) {- first := buildExtractedMessage(8, 42, "mem-1", msg, test.item, firstAttempt)- retry := buildExtractedMessage(9, 42, "mem-1", msg, test.item, retryAttempt)+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.Local)+ item := extractedMemory{+ MessageType: "fact",+ Content: "likes coffee",+ }- firstID, ok := first["id"].(string)- if !ok || firstID == "" {- t.Fatalf("first extracted message id = %#v, want a stable non-empty string", first["id"])- }- if got, ok := retry["id"].(string); !ok || got != firstID {- t.Fatalf("retry extracted message id = %#v, want %q", retry["id"], firstID)- }- })+ extracted := buildExtractedMessage(8, 42, "mem-1", msg, item, now)+ gotID, ok := extracted["id"].(string)+ if !ok || gotID != "mem-1_8" {+ t.Fatalf("extracted message id = %#v, want %q", extracted["id"], "mem-1_8")}}