Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(memory): align extracted message document id with message_id (#19326)

jay77721545753e248f8
diff --git a/internal/service/memory_extractor.go b/internal/service/memory_extractor.go
index 67273cbac5..fc63fa879d 100644
--- a/internal/service/memory_extractor.go
+++ b/internal/service/memory_extractor.go
@@ -36,7 +36,6 @@ package service
import (
"context"
- "crypto/sha256"
"encoding/json"
"errors"
"fmt"
@@ -247,30 +246,20 @@ func (s *MemoryMessageService) extractByLLM(ctx context.Context, mem *CreateMemo
// logical message fields are set here; the doc engine maps them to
// storage fields (including tokenization) at insert time.
func buildExtractedMessage(messageID, sourceID int64, memoryID string, msg MemoryMessage, item extractedMemory, now time.Time) map[string]any {
- validAt, hasExplicitValidAt := normalizeMemoryTime(item.ValidAt)
- if !hasExplicitValidAt {
+ validAt, ok := normalizeMemoryTime(item.ValidAt)
+ if !ok {
validAt = now.Format(memoryTimeLayout)
}
- invalidAt, hasExplicitInvalidAt := normalizeMemoryTime(item.InvalidAt)
- if strings.TrimSpace(item.InvalidAt) != "" {
- if !hasExplicitInvalidAt {
- invalidAt = now.Format(memoryTimeLayout)
- }
+ invalidAt, ok := normalizeMemoryTime(item.InvalidAt)
+ if strings.TrimSpace(item.InvalidAt) != "" && !ok {
+ invalidAt = now.Format(memoryTimeLayout)
}
var storedInvalidAt any
if invalidAt != "" {
storedInvalidAt = invalidAt
}
- fingerprintValidAt := ""
- if hasExplicitValidAt {
- fingerprintValidAt = validAt
- }
- fingerprintInvalidAt := ""
- if hasExplicitInvalidAt {
- fingerprintInvalidAt = invalidAt
- }
return map[string]any{
- "id": extractedMessageDocumentID(memoryID, sourceID, item.MessageType, item.Content, fingerprintValidAt, fingerprintInvalidAt),
+ "id": fmt.Sprintf("%s_%d", memoryID, messageID),
"message_id": messageID,
"message_type": item.MessageType,
"source_id": sourceID,
@@ -286,17 +275,6 @@ func buildExtractedMessage(messageID, sourceID int64, memoryID string, msg Memor
}
}
-// extractedMessageDocumentID returns the chunk-store id for an extracted item.
-// sourceID identifies the immutable raw message, while the normalized content
-// fingerprint distinguishes multiple extracts from that source. Only explicit,
-// parseable timestamps participate, so a runtime fallback cannot change the
-// identity on retry. Repeated task executions therefore upsert the same chunk
-// instead of creating duplicates.
-func extractedMessageDocumentID(memoryID string, sourceID int64, messageType, content, validAt, invalidAt string) string {
- fingerprint := sha256.Sum256([]byte(strings.Join([]string{messageType, content, validAt, invalidAt}, "\x00")))
- return fmt.Sprintf("%s_%d_%x", memoryID, sourceID, fingerprint[:8])
-}
-
// parseMemoryExtraction ports memory.utils.msg_util.get_json_result_from_llm_response
// plus the per-type flattening in extract_by_llm. Only the configured
// extract types are collected; unparseable responses yield an empty list.
diff --git a/internal/service/memory_extractor_test.go b/internal/service/memory_extractor_test.go
index 8357b10774..fd305ffc1e 100644
--- a/internal/service/memory_extractor_test.go
+++ b/internal/service/memory_extractor_test.go
@@ -99,45 +99,20 @@ func TestBuildExtractedMessageValidAtSemantics(t *testing.T) {
}
}
-// TestBuildExtractedMessageUsesStableDocumentIDAcrossRetries ensures fallback
-// timestamps do not become part of the chunk-store identity. A retry may run
-// at a different time and allocate a new logical message id, but it must still
-// overwrite the same extracted document.
-func TestBuildExtractedMessageUsesStableDocumentIDAcrossRetries(t *testing.T) {
+// TestBuildExtractedMessageUsesStandardDocumentID ensures extracted items
+// use the standard memoryID_messageID identity expected by GetMessageContent.
+func TestBuildExtractedMessageUsesStandardDocumentID(t *testing.T) {
msg := MemoryMessage{UserID: "u1", AgentID: "a1", SessionID: "s1"}
- firstAttempt := time.Date(2026, 8, 20, 10, 5, 0, 0, time.Local)
- retryAttempt := firstAttempt.Add(10 * time.Second)
-
- for _, test := range []struct {
- name string
- item extractedMemory
- }{
- {
- name: "omitted valid at",
- item: extractedMemory{MessageType: "fact", Content: "likes coffee"},
- },
- {
- name: "invalid timestamps",
- item: extractedMemory{
- MessageType: "fact",
- Content: "likes coffee",
- ValidAt: "not a timestamp",
- InvalidAt: "also not a timestamp",
- },
- },
- } {
- t.Run(test.name, func(t *testing.T) {
- first := buildExtractedMessage(8, 42, "mem-1", msg, test.item, firstAttempt)
- retry := buildExtractedMessage(9, 42, "mem-1", msg, test.item, retryAttempt)
+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.Local)
+ item := extractedMemory{
+ MessageType: "fact",
+ Content: "likes coffee",
+ }
- firstID, ok := first["id"].(string)
- if !ok || firstID == "" {
- t.Fatalf("first extracted message id = %#v, want a stable non-empty string", first["id"])
- }
- if got, ok := retry["id"].(string); !ok || got != firstID {
- t.Fatalf("retry extracted message id = %#v, want %q", retry["id"], firstID)
- }
- })
+ extracted := buildExtractedMessage(8, 42, "mem-1", msg, item, now)
+ gotID, ok := extracted["id"].(string)
+ if !ok || gotID != "mem-1_8" {
+ t.Fatalf("extracted message id = %#v, want %q", extracted["id"], "mem-1_8")
}
}