Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(memory): make Go memory extraction tasks durable and resumable (#19512)

taek10521ae263e565a
diff --git a/internal/common/task.go b/internal/common/task.go
index c8c2447cde..91429ba724 100644
--- a/internal/common/task.go
+++ b/internal/common/task.go
@@ -16,8 +16,6 @@
package common
-import "encoding/json"
-
const (
// TaskSubject is the NATS subject on which ingestion and memory tasks are
// published and consumed. Producer and consumer must reference this single
@@ -35,20 +33,17 @@ const (
// TaskTypeMemory is the async memory-extraction task type. Memory tasks
// share the tasks.RAGFLOW subject and the Ingestor's consumer + worker
// pool with ingestion tasks; handleAndExecute dispatches them by TaskType.
- // The memory-specific payload (message_dict/memory_id/source_id) is
- // carried in TaskMessage.Payload.
+ // Their TaskMessage is only a wake-up; input lives in memory_task.
TaskTypeMemory = "memory"
)
+// TaskMessage is a broker wake-up that identifies one durable task.
type TaskMessage struct {
TaskID string `json:"task_id" binding:"required"`
TaskType string `json:"task_type" binding:"required"`
- // Payload carries the task-specific body for non-ingestion task types
- // (e.g. the memory extraction payload). It is left empty for ingestion
- // tasks and old messages, so existing consumers are unaffected.
- Payload json.RawMessage `json:"payload,omitempty"`
}
+// TaskHandle controls settlement and heartbeat for a received task message.
type TaskHandle interface {
GetMessage() TaskMessage
Ack() error
diff --git a/internal/dao/database.go b/internal/dao/database.go
index af0fafc9be..940ed7bfbf 100644
--- a/internal/dao/database.go
+++ b/internal/dao/database.go
@@ -140,6 +140,7 @@ func InitDB(ctx context.Context, migrateDB bool) error {
&entity.SyncLogs{},
&entity.MCPServer{},
&entity.Memory{},
+ &entity.MemoryTask{},
&entity.Search{},
&entity.PipelineOperationLog{},
&entity.EvaluationDataset{},
@@ -183,7 +184,7 @@ func InitDB(ctx context.Context, migrateDB bool) error {
} else {
// Ensure Go-exclusive runtime tables exist even if the server starts without --migrate
if err = autoMigrateRuntimeModels(ctx, DB); err != nil {
- common.Warn("Failed to auto-migrate runtime models", zap.Error(err))
+ return fmt.Errorf("failed to auto-migrate runtime models: %w", err)
}
}
// Seed built-in agent templates so the Go backend can serve the
@@ -301,10 +302,15 @@ func autoMigrateRuntimeModels(ctx context.Context, db *gorm.DB) error {
goRuntimeModels := []interface{}{
&entity.IngestionTask{},
&entity.IngestionTaskLog{},
+ &entity.MemoryTask{},
}
for _, m := range goRuntimeModels {
if err := autoMigrateSafely(ctx, db, m); err != nil {
- return fmt.Errorf("failed to auto-migrate runtime model %T: %w", m, err)
+ tableName := fmt.Sprintf("%T", m)
+ if named, ok := m.(interface{ TableName() string }); ok {
+ tableName = named.TableName()
+ }
+ return fmt.Errorf("failed to auto-migrate runtime table %s: %w", tableName, err)
}
}
return nil
diff --git a/internal/dao/database_test.go b/internal/dao/database_test.go
index 204f4bac86..6214b6a4cf 100644
--- a/internal/dao/database_test.go
+++ b/internal/dao/database_test.go
@@ -18,6 +18,7 @@ package dao
import (
"context"
+ "strings"
"testing"
"ragflow/internal/entity"
@@ -26,7 +27,7 @@ import (
"gorm.io/gorm"
)
-func TestAutoMigrateRuntimeModelsCreatesIngestionTaskTables(t *testing.T) {
+func TestAutoMigrateRuntimeModelsCreatesGoRuntimeTables(t *testing.T) {
db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
if err != nil {
t.Fatalf("open sqlite: %v", err)
@@ -39,6 +40,9 @@ func TestAutoMigrateRuntimeModelsCreatesIngestionTaskTables(t *testing.T) {
if db.Migrator().HasTable(&entity.IngestionTaskLog{}) {
t.Fatal("expected ingestion_task_log to not exist initially")
}
+ if db.Migrator().HasTable(&entity.MemoryTask{}) {
+ t.Fatal("expected memory_task to not exist initially")
+ }
ctx := context.Background()
if err = autoMigrateRuntimeModels(ctx, db); err != nil {
@@ -52,9 +56,66 @@ func TestAutoMigrateRuntimeModelsCreatesIngestionTaskTables(t *testing.T) {
if !db.Migrator().HasTable(&entity.IngestionTaskLog{}) {
t.Fatal("expected ingestion_task_log to exist after autoMigrateRuntimeModels")
}
+ if !db.Migrator().HasTable(&entity.MemoryTask{}) {
+ t.Fatal("expected memory_task to exist after autoMigrateRuntimeModels")
+ }
+ if !db.Migrator().HasIndex(&entity.MemoryTask{}, "idx_memory_task_due") {
+ t.Fatal("expected memory_task due index to exist after autoMigrateRuntimeModels")
+ }
+
+ memoryTask := &entity.MemoryTask{
+ TaskID: "task-1",
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{
+ "user_id": "user-1",
+ "user_input": "remember this",
+ },
+ State: entity.MemoryTaskStatePending,
+ Extraction: entity.JSONSlice{
+ map[string]interface{}{"message_id": float64(7), "content": "remembered"},
+ },
+ LastError: "",
+ }
+ if err = db.Create(memoryTask).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+
+ var stored entity.MemoryTask
+ if err = db.First(&stored, "task_id = ?", memoryTask.TaskID).Error; err != nil {
+ t.Fatalf("load memory task: %v", err)
+ }
+ if stored.State != entity.MemoryTaskStatePending {
+ t.Fatalf("memory task state = %q, want %q", stored.State, entity.MemoryTaskStatePending)
+ }
+ if got := stored.Input["user_input"]; got != "remember this" {
+ t.Fatalf("memory task input user_input = %#v, want %q", got, "remember this")
+ }
+ if len(stored.Extraction) != 1 {
+ t.Fatalf("memory task extraction length = %d, want 1", len(stored.Extraction))
+ }
// Verify idempotency
if err = autoMigrateRuntimeModels(ctx, db); err != nil {
t.Fatalf("second autoMigrateRuntimeModels failed: %v", err)
}
}
+
+func TestAutoMigrateRuntimeModelsNamesFailingTable(t *testing.T) {
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatalf("get sql database: %v", err)
+ }
+ if err = sqlDB.Close(); err != nil {
+ t.Fatalf("close sql database: %v", err)
+ }
+
+ err = autoMigrateRuntimeModels(t.Context(), db)
+ if err == nil || !strings.Contains(err.Error(), "runtime table ingestion_task") {
+ t.Fatalf("autoMigrateRuntimeModels error = %v, want failing table name", err)
+ }
+}
diff --git a/internal/dao/memory.go b/internal/dao/memory.go
index 8b64d935aa..c93bf33cb6 100644
--- a/internal/dao/memory.go
+++ b/internal/dao/memory.go
@@ -273,8 +273,12 @@ func (dao *MemoryDAO) GetWithOwnerNameByID(ctx context.Context, db *gorm.DB, id
OwnerName *string `gorm:"column:owner_name"`
}
- if err := db.WithContext(ctx).Raw(querySQL, id).Scan(&rawResult).Error; err != nil {
- return nil, err
+ result := db.WithContext(ctx).Raw(querySQL, id).Scan(&rawResult)
+ if result.Error != nil {
+ return nil, result.Error
+ }
+ if result.RowsAffected == 0 {
+ return nil, gorm.ErrRecordNotFound
}
return &entity.MemoryListItem{
diff --git a/internal/dao/memory_task.go b/internal/dao/memory_task.go
new file mode 100644
index 0000000000..523f6d8ce0
--- /dev/null
+++ b/internal/dao/memory_task.go
@@ -0,0 +1,342 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "ragflow/internal/entity"
+
+ "gorm.io/gorm"
+ "gorm.io/gorm/clause"
+)
+
+var activeMemoryTaskStates = []entity.MemoryTaskState{
+ entity.MemoryTaskStatePending,
+ entity.MemoryTaskStateExtracted,
+ entity.MemoryTaskStateStored,
+}
+
+// MemoryTaskDAO persists durable memory extraction checkpoints and leases.
+type MemoryTaskDAO struct{}
+
+// NewMemoryTaskDAO creates a memory task DAO.
+func NewMemoryTaskDAO() *MemoryTaskDAO { return &MemoryTaskDAO{} }
+
+// CreateWithTask creates the UI task and its durable execution record in one
+// transaction so workers never observe only one side of the pair.
+func (d *MemoryTaskDAO) CreateWithTask(ctx context.Context, db *gorm.DB, task *entity.Task, memoryTask *entity.MemoryTask) error {
+ if db == nil {
+ return errors.New("memory task: nil database")
+ }
+ if task == nil || memoryTask == nil {
+ return errors.New("memory task: task and execution record are required")
+ }
+ if task.ID == "" || memoryTask.TaskID == "" || task.ID != memoryTask.TaskID {
+ return errors.New("memory task: task ids must be non-empty and equal")
+ }
+
+ return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := tx.Create(task).Error; err != nil {
+ return err
+ }
+ return tx.Create(memoryTask).Error
+ })
+}
+
+// GetByID returns one durable memory task.
+func (d *MemoryTaskDAO) GetByID(ctx context.Context, db *gorm.DB, taskID string) (*entity.MemoryTask, error) {
+ if db == nil {
+ return nil, errors.New("memory task: nil database")
+ }
+ var task entity.MemoryTask
+ if err := db.WithContext(ctx).Where("task_id = ?", taskID).First(&task).Error; err != nil {
+ return nil, err
+ }
+ return &task, nil
+}
+
+// Claim acquires an expired or unowned due task and increments its attempt
+// count. It returns the current row with acquired=false when the task is not
+// due, is terminal, or already has a live lease.
+func (d *MemoryTaskDAO) Claim(ctx context.Context, db *gorm.DB, taskID, owner string, now time.Time, leaseTTL time.Duration) (*entity.MemoryTask, bool, error) {
+ if db == nil {
+ return nil, false, errors.New("memory task: nil database")
+ }
+ if taskID == "" || owner == "" {
+ return nil, false, errors.New("memory task: task id and lease owner are required")
+ }
+ if leaseTTL <= 0 {
+ return nil, false, errors.New("memory task: lease TTL must be positive")
+ }
+
+ var (
+ task entity.MemoryTask
+ acquired bool
+ )
+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ query := tx.WithContext(ctx)
+ if tx.Dialector.Name() != "sqlite" {
+ query = query.Clauses(clause.Locking{Strength: "UPDATE"})
+ }
+ if err := query.Where("task_id = ?", taskID).First(&task).Error; err != nil {
+ return err
+ }
+ if !isActiveMemoryTaskState(task.State) ||
+ (task.NextRetryAt != nil && task.NextRetryAt.After(now)) ||
+ (task.LeaseExpiresAt != nil && task.LeaseExpiresAt.After(now)) {
+ return nil
+ }
+
+ expiresAt := now.Add(leaseTTL)
+ result := tx.WithContext(ctx).Model(&entity.MemoryTask{}).
+ Where("task_id = ?", taskID).
+ Updates(map[string]any{
+ "attempt_count": gorm.Expr("attempt_count + 1"),
+ "lease_owner": owner,
+ "lease_expires_at": expiresAt,
+ "next_retry_at": nil,
+ })
+ if result.Error != nil {
+ return result.Error
+ }
+ if result.RowsAffected != 1 {
+ return nil
+ }
+ acquired = true
+ return tx.WithContext(ctx).Where("task_id = ?", taskID).First(&task).Error
+ })
+ if err != nil {
+ return nil, false, err
+ }
+ return &task, acquired, err
+}
+
+// RenewLease extends a live lease held by owner.
+func (d *MemoryTaskDAO) RenewLease(ctx context.Context, db *gorm.DB, taskID, owner string, now time.Time, leaseTTL time.Duration) (bool, error) {
+ if db == nil {
+ return false, errors.New("memory task: nil database")
+ }
+ if taskID == "" || owner == "" || leaseTTL <= 0 {
+ return false, errors.New("memory task: task id, lease owner, and positive TTL are required")
+ }
+
+ result := db.WithContext(ctx).Model(&entity.MemoryTask{}).
+ Where("task_id = ? AND state IN ? AND lease_owner = ? AND lease_expires_at > ?", taskID, activeMemoryTaskStates, owner, now).
+ Update("lease_expires_at", now.Add(leaseTTL))
+ return result.RowsAffected == 1, result.Error
+}
+
+// PersistExtraction advances a leased task from pending to extracted and
+// stores the materialized LLM result used by later retries.
+func (d *MemoryTaskDAO) PersistExtraction(ctx context.Context, db *gorm.DB, taskID, owner string, now time.Time, extraction entity.JSONSlice) (bool, error) {
+ return d.transition(ctx, db, taskID, owner, now, entity.MemoryTaskStatePending, entity.MemoryTaskStateExtracted, map[string]any{
+ "extraction": extraction,
+ "last_error": "",
+ })
+}
+
+// MarkStored advances a leased task after its extracted chunks are stored.
+func (d *MemoryTaskDAO) MarkStored(ctx context.Context, db *gorm.DB, taskID, owner string, now time.Time) (bool, error) {
+ return d.transition(ctx, db, taskID, owner, now, entity.MemoryTaskStateExtracted, entity.MemoryTaskStateStored, map[string]any{
+ "last_error": "",
+ })
+}
+
+// ScheduleRetry records the next execution time and releases the current
+// lease. The task remains at its last durable checkpoint.
+func (d *MemoryTaskDAO) ScheduleRetry(ctx context.Context, db *gorm.DB, taskID, owner string, now, nextRetryAt time.Time, lastError string) (bool, error) {
+ if db == nil {
+ return false, errors.New("memory task: nil database")
+ }
+ if taskID == "" || owner == "" {
+ return false, errors.New("memory task: task id and lease owner are required")
+ }
+ result := db.WithContext(ctx).Model(&entity.MemoryTask{}).
+ Where("task_id = ? AND state IN ? AND lease_owner = ? AND lease_expires_at > ?", taskID, activeMemoryTaskStates, owner, now).
+ Updates(map[string]any{
+ "next_retry_at": nextRetryAt,
+ "lease_owner": "",
+ "lease_expires_at": nil,
+ "last_error": lastError,
+ })
+ return result.RowsAffected == 1, result.Error
+}
+
+// MarkFailed records a terminal failure and projects it onto the generic task
+// row when that row still exists.
+func (d *MemoryTaskDAO) MarkFailed(ctx context.Context, db *gorm.DB, taskID, owner, progressMsg string, now time.Time) (bool, error) {
+ if db == nil {
+ return false, errors.New("memory task: nil database")
+ }
+ if taskID == "" || owner == "" {
+ return false, errors.New("memory task: task id and lease owner are required")
+ }
+ var updated bool
+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ result := tx.WithContext(ctx).Model(&entity.MemoryTask{}).
+ Where("task_id = ? AND state IN ? AND lease_owner = ? AND lease_expires_at > ?", taskID, activeMemoryTaskStates, owner, now).
+ Updates(map[string]any{
+ "state": entity.MemoryTaskStateFailed,
+ "next_retry_at": nil,
+ "lease_owner": "",
+ "lease_expires_at": nil,
+ "last_error": progressMsg,
+ })
+ if result.Error != nil || result.RowsAffected != 1 {
+ return result.Error
+ }
+ updated = true
+ return tx.WithContext(ctx).Model(&entity.Task{}).Where("id = ?", taskID).Updates(map[string]any{
+ "progress": -1,
+ "progress_msg": progressMsg,
+ }).Error
+ })
+ if err != nil {
+ return false, err
+ }
+ return updated, nil
+}
+
+// MarkUnclaimedFailed records a terminal failure before a newly created task
+// has been published or claimed, and updates its UI projection atomically.
+func (d *MemoryTaskDAO) MarkUnclaimedFailed(ctx context.Context, db *gorm.DB, taskID, progressMsg string) (bool, error) {
+ if db == nil {
+ return false, errors.New("memory task: nil database")
+ }
+ if taskID == "" {
+ return false, errors.New("memory task: task id is required")
+ }
+
+ var updated bool
+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ result := tx.WithContext(ctx).Model(&entity.MemoryTask{}).
+ Where("task_id = ? AND state = ? AND lease_owner = '' AND lease_expires_at IS NULL", taskID, entity.MemoryTaskStatePending).
+ Updates(map[string]any{
+ "state": entity.MemoryTaskStateFailed,
+ "next_retry_at": nil,
+ "last_error": progressMsg,
+ })
+ if result.Error != nil {
+ return result.Error
+ }
+ if result.RowsAffected != 1 {
+ return nil
+ }
+
+ if err := tx.WithContext(ctx).Model(&entity.Task{}).Where("id = ?", taskID).Updates(map[string]any{
+ "progress": -1,
+ "progress_msg": progressMsg,
+ }).Error; err != nil {
+ return err
+ }
+ updated = true
+ return nil
+ })
+ if err != nil {
+ return false, err
+ }
+ return updated, nil
+}
+
+// Complete atomically advances a leased stored task to completed and updates
+// the generic task progress projection when it still exists.
+func (d *MemoryTaskDAO) Complete(ctx context.Context, db *gorm.DB, taskID, owner, progressMsg string, now time.Time) (bool, error) {
+ if db == nil {
+ return false, errors.New("memory task: nil database")
+ }
+ if taskID == "" || owner == "" {
+ return false, errors.New("memory task: task id and lease owner are required")
+ }
+ var completed bool
+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ result := tx.WithContext(ctx).Model(&entity.MemoryTask{}).
+ Where("task_id = ? AND state = ? AND lease_owner = ? AND lease_expires_at > ?", taskID, entity.MemoryTaskStateStored, owner, now).
+ Updates(map[string]any{
+ "state": entity.MemoryTaskStateCompleted,
+ "next_retry_at": nil,
+ "lease_owner": "",
+ "lease_expires_at": nil,
+ "last_error": "",
+ })
+ if result.Error != nil {
+ return result.Error
+ }
+ if result.RowsAffected != 1 {
+ return nil
+ }
+
+ progress := tx.WithContext(ctx).Model(&entity.Task{}).Where("id = ?", taskID).Updates(map[string]any{
+ "progress": 1.0,
+ "progress_msg": progressMsg,
+ })
+ if progress.Error != nil {
+ return progress.Error
+ }
+ completed = true
+ return nil
+ })
+ if err != nil {
+ return false, err
+ }
+ return completed, nil
+}
+
+// ListDue returns non-terminal tasks whose retry time and lease allow another
+// wake-up to be published.
+func (d *MemoryTaskDAO) ListDue(ctx context.Context, db *gorm.DB, now time.Time, limit int) ([]*entity.MemoryTask, error) {
+ if db == nil {
+ return nil, errors.New("memory task: nil database")
+ }
+ if limit <= 0 {
+ return nil, errors.New("memory task: positive list limit is required")
+ }
+
+ var tasks []*entity.MemoryTask
+ err := db.WithContext(ctx).
+ Where("state IN ?", activeMemoryTaskStates).
+ Where("next_retry_at IS NULL OR next_retry_at <= ?", now).
+ Where("lease_expires_at IS NULL OR lease_expires_at <= ?", now).
+ Order("next_retry_at ASC, create_time ASC, task_id ASC").
+ Limit(limit).
+ Find(&tasks).Error
+ return tasks, err
+}
+
+// transition applies a lease-guarded checkpoint change.
+func (d *MemoryTaskDAO) transition(ctx context.Context, db *gorm.DB, taskID, owner string, now time.Time, from, to entity.MemoryTaskState, updates map[string]any) (bool, error) {
+ if db == nil {
+ return false, errors.New("memory task: nil database")
+ }
+ if taskID == "" || owner == "" {
+ return false, errors.New("memory task: task id and lease owner are required")
+ }
+ updates["state"] = to
+ result := db.WithContext(ctx).Model(&entity.MemoryTask{}).
+ Where("task_id = ? AND state = ? AND lease_owner = ? AND lease_expires_at > ?", taskID, from, owner, now).
+ Updates(updates)
+ return result.RowsAffected == 1, result.Error
+}
+
+// isActiveMemoryTaskState reports whether a task can still be executed.
+func isActiveMemoryTaskState(state entity.MemoryTaskState) bool {
+ return state == entity.MemoryTaskStatePending ||
+ state == entity.MemoryTaskStateExtracted ||
+ state == entity.MemoryTaskStateStored
+}
diff --git a/internal/dao/memory_task_test.go b/internal/dao/memory_task_test.go
new file mode 100644
index 0000000000..b40503a445
--- /dev/null
+++ b/internal/dao/memory_task_test.go
@@ -0,0 +1,342 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package dao
+
+import (
+ "testing"
+ "time"
+
+ "ragflow/internal/entity"
+
+ "github.com/glebarez/sqlite"
+ "gorm.io/gorm"
+)
+
+// setupMemoryTaskTestDB creates an isolated SQLite store for DAO tests.
+func setupMemoryTaskTestDB(t *testing.T) *gorm.DB {
+ t.Helper()
+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})
+ if err != nil {
+ t.Fatalf("open sqlite: %v", err)
+ }
+ if err = db.AutoMigrate(&entity.Task{}, &entity.MemoryTask{}); err != nil {
+ t.Fatalf("auto-migrate memory tasks: %v", err)
+ }
+ sqlDB, err := db.DB()
+ if err != nil {
+ t.Fatalf("get sql database: %v", err)
+ }
+ sqlDB.SetMaxOpenConns(1)
+ return db
+}
+
+// newMemoryTaskPair returns matching UI and execution records.
+func newMemoryTaskPair(taskID string) (*entity.Task, *entity.MemoryTask) {
+ progressMsg := ""
+ task := &entity.Task{
+ ID: taskID,
+ DocID: "memory-1",
+ TaskType: "memory",
+ ProgressMsg: &progressMsg,
+ }
+ memoryTask := &entity.MemoryTask{
+ TaskID: taskID,
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{
+ "user_id": "user-1",
+ "user_input": "remember this",
+ },
+ State: entity.MemoryTaskStatePending,
+ LastError: "",
+ }
+ return task, memoryTask
+}
+
+// TestMemoryTaskDAOCreateWithTaskIsAtomic verifies a failed execution-record
+// insert rolls back the generic task insert.
+func TestMemoryTaskDAOCreateWithTaskIsAtomic(t *testing.T) {
+ db := setupMemoryTaskTestDB(t)
+ dao := NewMemoryTaskDAO()
+
+ _, existing := newMemoryTaskPair("task-1")
+ if err := db.Create(existing).Error; err != nil {
+ t.Fatalf("seed memory task: %v", err)
+ }
+ task, memoryTask := newMemoryTaskPair("task-1")
+ if err := dao.CreateWithTask(t.Context(), db, task, memoryTask); err == nil {
+ t.Fatal("CreateWithTask error = nil, want duplicate-key error")
+ }
+
+ var taskCount int64
+ if err := db.Model(&entity.Task{}).Where("id = ?", task.ID).Count(&taskCount).Error; err != nil {
+ t.Fatalf("count generic tasks: %v", err)
+ }
+ if taskCount != 0 {
+ t.Fatalf("generic task count = %d, want transaction rollback", taskCount)
+ }
+}
+
+// TestMemoryTaskDAOClaimRenewAndRetry verifies lease exclusion, renewal, and
+// retry-time admission.
+func TestMemoryTaskDAOClaimRenewAndRetry(t *testing.T) {
+ db := setupMemoryTaskTestDB(t)
+ dao := NewMemoryTaskDAO()
+ task, memoryTask := newMemoryTaskPair("task-1")
+ if err := dao.CreateWithTask(t.Context(), db, task, memoryTask); err != nil {
+ t.Fatalf("CreateWithTask: %v", err)
+ }
+
+ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
+ claimed, acquired, err := dao.Claim(t.Context(), db, task.ID, "worker-1", now, time.Minute)
+ if err != nil || !acquired {
+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)
+ }
+ if claimed.AttemptCount != 1 || claimed.LeaseOwner != "worker-1" {
+ t.Fatalf("claimed task = %+v, want attempt 1 owned by worker-1", claimed)
+ }
+
+ if _, acquired, err = dao.Claim(t.Context(), db, task.ID, "worker-2", now.Add(30*time.Second), time.Minute); err != nil || acquired {
+ t.Fatalf("duplicate Claim acquired=%v err=%v, want live lease rejection", acquired, err)
+ }
+ if renewed, renewErr := dao.RenewLease(t.Context(), db, task.ID, "worker-1", now.Add(30*time.Second), time.Minute); renewErr != nil || !renewed {
+ t.Fatalf("RenewLease renewed=%v err=%v, want renewed", renewed, renewErr)
+ }
+
+ retryAt := now.Add(2 * time.Minute)
+ if scheduled, scheduleErr := dao.ScheduleRetry(t.Context(), db, task.ID, "worker-1", now.Add(30*time.Second), retryAt, "temporary failure"); scheduleErr != nil || !scheduled {
+ t.Fatalf("ScheduleRetry scheduled=%v err=%v, want scheduled", scheduled, scheduleErr)
+ }
+ if _, acquired, err = dao.Claim(t.Context(), db, task.ID, "worker-2", retryAt.Add(-time.Second), time.Minute); err != nil || acquired {
+ t.Fatalf("early retry Claim acquired=%v err=%v, want not due", acquired, err)
+ }
+ claimed, acquired, err = dao.Claim(t.Context(), db, task.ID, "worker-2", retryAt, time.Minute)
+ if err != nil || !acquired {
+ t.Fatalf("due retry Claim acquired=%v err=%v, want acquired", acquired, err)
+ }
+ if claimed.AttemptCount != 2 || claimed.LeaseOwner != "worker-2" {
+ t.Fatalf("retried task = %+v, want attempt 2 owned by worker-2", claimed)
+ }
+}
+
+// TestMemoryTaskDAOClaimAfterLeaseExpiry verifies a crashed worker cannot
+// strand a task after its durable lease expires.
+func TestMemoryTaskDAOClaimAfterLeaseExpiry(t *testing.T) {
+ db := setupMemoryTaskTestDB(t)
+ dao := NewMemoryTaskDAO()
+ task, memoryTask := newMemoryTaskPair("task-expired-lease")
+ if err := dao.CreateWithTask(t.Context(), db, task, memoryTask); err != nil {
+ t.Fatalf("CreateWithTask: %v", err)
+ }
+
+ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
+ leaseTTL := time.Minute
+ if _, acquired, err := dao.Claim(t.Context(), db, task.ID, "worker-1", now, leaseTTL); err != nil || !acquired {
+ t.Fatalf("first Claim acquired=%v err=%v, want acquired", acquired, err)
+ }
+ if _, acquired, err := dao.Claim(t.Context(), db, task.ID, "worker-2", now.Add(leaseTTL-time.Second), leaseTTL); err != nil || acquired {
+ t.Fatalf("live-lease Claim acquired=%v err=%v, want rejected", acquired, err)
+ }
+ claimed, acquired, err := dao.Claim(t.Context(), db, task.ID, "worker-2", now.Add(leaseTTL), leaseTTL)
+ if err != nil || !acquired {
+ t.Fatalf("expired-lease Claim acquired=%v err=%v, want acquired", acquired, err)
+ }
+ if claimed.LeaseOwner != "worker-2" || claimed.AttemptCount != 2 {
+ t.Fatalf("reclaimed task owner/attempt = %q/%d, want worker-2/2", claimed.LeaseOwner, claimed.AttemptCount)
+ }
+}
+
+// TestMemoryTaskDAOExpiredLeaseCannotRecordFailure verifies a stale worker
+// cannot schedule a retry or mark a task failed after its lease expires.
+func TestMemoryTaskDAOExpiredLeaseCannotRecordFailure(t *testing.T) {
+ db := setupMemoryTaskTestDB(t)
+ dao := NewMemoryTaskDAO()
+ task, memoryTask := newMemoryTaskPair("task-expired-failure")
+ if err := dao.CreateWithTask(t.Context(), db, task, memoryTask); err != nil {
+ t.Fatalf("CreateWithTask: %v", err)
+ }
+
+ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
+ leaseTTL := time.Minute
+ if _, acquired, err := dao.Claim(t.Context(), db, task.ID, "worker-1", now, leaseTTL); err != nil || !acquired {
+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)
+ }
+ expiredAt := now.Add(leaseTTL)
+ if scheduled, err := dao.ScheduleRetry(t.Context(), db, task.ID, "worker-1", expiredAt, expiredAt.Add(time.Minute), "temporary failure"); err != nil || scheduled {
+ t.Fatalf("ScheduleRetry scheduled=%v err=%v, want expired lease rejection", scheduled, err)
+ }
+ if failed, err := dao.MarkFailed(t.Context(), db, task.ID, "worker-1", "permanent failure", expiredAt); err != nil || failed {
+ t.Fatalf("MarkFailed failed=%v err=%v, want expired lease rejection", failed, err)
+ }
+
+ stored, err := dao.GetByID(t.Context(), db, task.ID)
+ if err != nil {
+ t.Fatalf("GetByID: %v", err)
+ }
+ if stored.State != entity.MemoryTaskStatePending || stored.LeaseOwner != "worker-1" || stored.NextRetryAt != nil || stored.LastError != "" {
+ t.Fatalf("memory task changed after expired-lease updates: %+v", stored)
+ }
+ var genericTask entity.Task
+ if err = db.First(&genericTask, "id = ?", task.ID).Error; err != nil {
+ t.Fatalf("load generic task: %v", err)
+ }
+ if genericTask.Progress != 0 {
+ t.Fatalf("generic task progress = %v, want unchanged", genericTask.Progress)
+ }
+}
+
+// TestMemoryTaskDAOCheckpointAndComplete verifies the complete durable state
+// sequence and final UI progress projection.
+func TestMemoryTaskDAOCheckpointAndComplete(t *testing.T) {
+ db := setupMemoryTaskTestDB(t)
+ dao := NewMemoryTaskDAO()
+ task, memoryTask := newMemoryTaskPair("task-1")
+ if err := dao.CreateWithTask(t.Context(), db, task, memoryTask); err != nil {
+ t.Fatalf("CreateWithTask: %v", err)
+ }
+
+ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
+ if _, acquired, err := dao.Claim(t.Context(), db, task.ID, "worker-1", now, time.Minute); err != nil || !acquired {
+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)
+ }
+ extraction := entity.JSONSlice{
+ map[string]interface{}{"message_id": float64(7), "content": "remembered"},
+ }
+ if updated, err := dao.PersistExtraction(t.Context(), db, task.ID, "worker-1", now, extraction); err != nil || !updated {
+ t.Fatalf("PersistExtraction updated=%v err=%v, want updated", updated, err)
+ }
+ if updated, err := dao.MarkStored(t.Context(), db, task.ID, "worker-1", now); err != nil || !updated {
+ t.Fatalf("MarkStored updated=%v err=%v, want updated", updated, err)
+ }
+ if completed, err := dao.Complete(t.Context(), db, task.ID, "worker-1", "complete", now); err != nil || !completed {
+ t.Fatalf("Complete completed=%v err=%v, want completed", completed, err)
+ }
+
+ stored, err := dao.GetByID(t.Context(), db, task.ID)
+ if err != nil {
+ t.Fatalf("GetByID: %v", err)
+ }
+ if stored.State != entity.MemoryTaskStateCompleted || stored.LeaseOwner != "" || stored.LeaseExpiresAt != nil {
+ t.Fatalf("completed memory task = %+v", stored)
+ }
+ var genericTask entity.Task
+ if err = db.First(&genericTask, "id = ?", task.ID).Error; err != nil {
+ t.Fatalf("load generic task: %v", err)
+ }
+ if genericTask.Progress != 1 || genericTask.ProgressMsg == nil || *genericTask.ProgressMsg != "complete" {
+ t.Fatalf("generic task progress = %v message=%v, want 1/complete", genericTask.Progress, genericTask.ProgressMsg)
+ }
+}
+
+// TestMemoryTaskDAOCompleteRollsBackWhenProgressUpdateFails verifies the
+// stored checkpoint remains retryable when the UI projection cannot be
+// committed in the same transaction.
+func TestMemoryTaskDAOCompleteRollsBackWhenProgressUpdateFails(t *testing.T) {
+ db := setupMemoryTaskTestDB(t)
+ dao := NewMemoryTaskDAO()
+ task, memoryTask := newMemoryTaskPair("task-completion-failure")
+ memoryTask.State = entity.MemoryTaskStateStored
+ memoryTask.LeaseOwner = "worker-1"
+ expiresAt := time.Date(2026, 9, 10, 12, 1, 0, 0, time.UTC)
+ memoryTask.LeaseExpiresAt = &expiresAt
+ if err := dao.CreateWithTask(t.Context(), db, task, memoryTask); err != nil {
+ t.Fatalf("CreateWithTask: %v", err)
+ }
+ if err := db.Exec(`
+ CREATE TRIGGER fail_memory_task_progress_update
+ BEFORE UPDATE ON task
+ BEGIN
+ SELECT RAISE(FAIL, 'forced progress update failure');
+ END
+ `).Error; err != nil {
+ t.Fatalf("create update trigger: %v", err)
+ }
+
+ completed, err := dao.Complete(t.Context(), db, task.ID, "worker-1", "complete", expiresAt.Add(-time.Second))
+ if err == nil || completed {
+ t.Fatalf("Complete completed=%v err=%v, want transaction failure", completed, err)
+ }
+ stored, getErr := dao.GetByID(t.Context(), db, task.ID)
+ if getErr != nil {
+ t.Fatalf("GetByID: %v", getErr)
+ }
+ if stored.State != entity.MemoryTaskStateStored || stored.LeaseOwner != "worker-1" || stored.LeaseExpiresAt == nil {
+ t.Fatalf("memory task after rollback = %+v, want stored checkpoint and original lease", stored)
+ }
+ var genericTask entity.Task
+ if getErr = db.First(&genericTask, "id = ?", task.ID).Error; getErr != nil {
+ t.Fatalf("load generic task: %v", getErr)
+ }
+ if genericTask.Progress != 0 {
+ t.Fatalf("generic task progress = %v, want unchanged", genericTask.Progress)
+ }
+}
+
+// TestMemoryTaskDAOCompleteWithoutGenericTask verifies a missing optional UI
+// projection does not roll back the authoritative durable completion state.
+func TestMemoryTaskDAOCompleteWithoutGenericTask(t *testing.T) {
+ db := setupMemoryTaskTestDB(t)
+ dao := NewMemoryTaskDAO()
+ _, memoryTask := newMemoryTaskPair("task-1")
+ memoryTask.State = entity.MemoryTaskStateStored
+ memoryTask.LeaseOwner = "worker-1"
+ expiresAt := time.Date(2026, 9, 10, 12, 1, 0, 0, time.UTC)
+ memoryTask.LeaseExpiresAt = &expiresAt
+ if err := db.Create(memoryTask).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+
+ completed, err := dao.Complete(t.Context(), db, memoryTask.TaskID, "worker-1", "complete", expiresAt.Add(-time.Second))
+ if err != nil || !completed {
+ t.Fatalf("Complete completed=%v err=%v, want completed", completed, err)
+ }
+ stored, getErr := dao.GetByID(t.Context(), db, memoryTask.TaskID)
+ if getErr != nil {
+ t.Fatalf("GetByID: %v", getErr)
+ }
+ if stored.State != entity.MemoryTaskStateCompleted || stored.LeaseOwner != "" || stored.LeaseExpiresAt != nil {
+ t.Fatalf("completed memory task = %+v", stored)
+ }
+}
+
+// TestMemoryTaskDAOListDueExcludesFutureLeasedAndTerminalTasks verifies the
+// reconciler query returns only executable rows.
+func TestMemoryTaskDAOListDueExcludesFutureLeasedAndTerminalTasks(t *testing.T) {
+ db := setupMemoryTaskTestDB(t)
+ dao := NewMemoryTaskDAO()
+ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
+ future := now.Add(time.Minute)
+
+ for _, task := range []*entity.MemoryTask{
+ {TaskID: "due", MemoryID: "memory-1", SourceID: 1, Input: entity.JSONMap{}, State: entity.MemoryTaskStatePending, LastError: ""},
+ {TaskID: "future", MemoryID: "memory-1", SourceID: 2, Input: entity.JSONMap{}, State: entity.MemoryTaskStatePending, NextRetryAt: &future, LastError: ""},
+ {TaskID: "leased", MemoryID: "memory-1", SourceID: 3, Input: entity.JSONMap{}, State: entity.MemoryTaskStateExtracted, LeaseOwner: "worker-1", LeaseExpiresAt: &future, LastError: ""},
+ {TaskID: "completed", MemoryID: "memory-1", SourceID: 4, Input: entity.JSONMap{}, State: entity.MemoryTaskStateCompleted, LastError: ""},
+ } {
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create task %s: %v", task.TaskID, err)
+ }
+ }
+
+ tasks, err := dao.ListDue(t.Context(), db, now, 10)
+ if err != nil {
+ t.Fatalf("ListDue: %v", err)
+ }
+ if len(tasks) != 1 || tasks[0].TaskID != "due" {
+ t.Fatalf("ListDue = %+v, want only due task", tasks)
+ }
+}
diff --git a/internal/dao/task.go b/internal/dao/task.go
index 18624897e6..bc0251a06f 100644
--- a/internal/dao/task.go
+++ b/internal/dao/task.go
@@ -18,6 +18,8 @@ package dao
import (
"context"
+
+ "ragflow/internal/common"
"ragflow/internal/entity"
"gorm.io/gorm"
@@ -62,6 +64,18 @@ func (dao *TaskDAO) UpdateProgress(ctx context.Context, db *gorm.DB, id string,
}).Error
}
+// MarkActiveMemoryTaskFailed marks an unfinished memory task's UI projection
+// as failed without overwriting a task that completed or failed concurrently.
+func (dao *TaskDAO) MarkActiveMemoryTaskFailed(ctx context.Context, db *gorm.DB, id, progressMsg string) (bool, error) {
+ result := db.WithContext(ctx).Model(&entity.Task{}).
+ Where("id = ? AND task_type = ? AND progress >= 0 AND progress < 1", id, common.TaskTypeMemory).
+ Updates(map[string]any{
+ "progress": -1,
+ "progress_msg": progressMsg,
+ })
+ return result.RowsAffected == 1, result.Error
+}
+
// DeleteIngestionTasksByDocIDs deletes ingestion tasks by document IDs (hard delete)
func (dao *TaskDAO) DeleteIngestionTasksByDocIDs(ctx context.Context, db *gorm.DB, docIDs []string) (int64, error) {
if len(docIDs) == 0 {
diff --git a/internal/entity/memory_task.go b/internal/entity/memory_task.go
new file mode 100644
index 0000000000..e7650f6623
--- /dev/null
+++ b/internal/entity/memory_task.go
@@ -0,0 +1,51 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package entity
+
+import "time"
+
+// MemoryTaskState is a durable checkpoint in memory extraction.
+type MemoryTaskState string
+
+const (
+ MemoryTaskStatePending MemoryTaskState = "pending"
+ MemoryTaskStateExtracted MemoryTaskState = "extracted"
+ MemoryTaskStateStored MemoryTaskState = "stored"
+ MemoryTaskStateCompleted MemoryTaskState = "completed"
+ MemoryTaskStateFailed MemoryTaskState = "failed"
+)
+
+// MemoryTask stores the authoritative execution state for an asynchronous
+// memory extraction task. Task.Progress remains a UI projection and is not an
+// execution checkpoint.
+type MemoryTask struct {
+ TaskID string `gorm:"column:task_id;primaryKey;size:32" json:"task_id"`
+ MemoryID string `gorm:"column:memory_id;size:32;not null;index" json:"memory_id"`
+ SourceID int64 `gorm:"column:source_id;not null" json:"source_id"`
+ Input JSONMap `gorm:"column:input;type:longtext;not null" json:"input"`
+ State MemoryTaskState `gorm:"column:state;size:16;not null;default:'pending';index:idx_memory_task_due,priority:1" json:"state"`
+ Extraction JSONSlice `gorm:"column:extraction;type:longtext" json:"extraction,omitempty"`
+ NextRetryAt *time.Time `gorm:"column:next_retry_at;default:null;index:idx_memory_task_due,priority:2" json:"next_retry_at,omitempty"`
+ AttemptCount int `gorm:"column:attempt_count;not null;default:0" json:"attempt_count"`
+ LeaseOwner string `gorm:"column:lease_owner;size:128;not null;default:''" json:"lease_owner"`
+ LeaseExpiresAt *time.Time `gorm:"column:lease_expires_at;default:null;index" json:"lease_expires_at,omitempty"`
+ LastError string `gorm:"column:last_error;type:text;not null" json:"last_error"`
+ BaseModel
+}
+
+// TableName returns the durable memory task table name.
+func (MemoryTask) TableName() string { return "memory_task" }
diff --git a/internal/ingestion/service/execute_memory_task_panic_test.go b/internal/ingestion/service/execute_memory_task_panic_test.go
index 45cb9ac82d..2cc0065d2a 100644
--- a/internal/ingestion/service/execute_memory_task_panic_test.go
+++ b/internal/ingestion/service/execute_memory_task_panic_test.go
@@ -24,7 +24,7 @@ import (
// so it does not depend on a live DB or a real MemoryMessageService. The contract
// asserted:
// - executeMemoryTask returns normally (no panic propagates to the caller/worker)
-// - the MQ message is Nacked so the broker redelivers it (never silently dropped)
+// - the MQ message is left unsettled for durable recovery
func TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {
_ = testutil.SetupTestDB(t) // ensure DB helpers are initialized; not exercised here
@@ -34,15 +34,12 @@ func TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {
ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
// Inject a panicking memory runner through the same seam used in production
// (defaultRunMemoryTask calls memorySvc.HandleSaveToMemoryTask).
- ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {
+ ingestor.runMemoryTask = func(_ context.Context, _, _ string) (servicepkg.MemoryTaskDisposition, error) {
panic("simulated memory extraction panic")
}
handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-1", TaskType: common.TaskTypeMemory}}
- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-panic-1", map[string]any{
- "memory_id": "mem-p", "source_id": 1,
- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},
- }, handle)
+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-panic-1", handle)
// If the panic is not recovered, this call itself will panic and the test
// fails. We also guard with a recover here only to convert a propagated
@@ -56,8 +53,8 @@ func TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {
ingestor.executeMemoryTask(context.Background(), taskCtx)
}()
- if handle.nacks.Load() != 1 || handle.acks.Load() != 0 {
- t.Fatalf("panicking memory task: expected 0 Ack/1 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ if handle.nacks.Load() != 0 || handle.acks.Load() != 0 {
+ t.Fatalf("panicking memory task: expected unsettled handle, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
}
}
@@ -73,7 +70,7 @@ func TestHandleAndExecute_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {
ingestor := NewIngestor("test", 1, []string{"pdf"})
ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
- ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {
+ ingestor.runMemoryTask = func(_ context.Context, _, _ string) (servicepkg.MemoryTaskDisposition, error) {
panic("simulated memory extraction panic")
}
ingestor.runDocumentTask = func(ctx context.Context, _ *entity.IngestionTask) error {
@@ -83,7 +80,6 @@ func TestHandleAndExecute_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {
memHandle := &fakeTaskHandle{msg: common.TaskMessage{
TaskID: "mem-panic-2",
TaskType: common.TaskTypeMemory,
- Payload: []byte(`{"memory_id":"mem-p2","source_id":1,"message_dict":{"user_id":"u","agent_id":"a","session_id":"s"}}`),
}}
ingestor.handleAndExecute(memHandle)
@@ -91,8 +87,8 @@ func TestHandleAndExecute_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {
docHandle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeIngestionTask}}
ingestor.handleAndExecute(docHandle)
- if memHandle.nacks.Load() != 1 || memHandle.acks.Load() != 0 {
- t.Fatalf("poison memory task: expected 0 Ack/1 Nack, got acks=%d nacks=%d", memHandle.acks.Load(), memHandle.nacks.Load())
+ if memHandle.nacks.Load() != 0 || memHandle.acks.Load() != 0 {
+ t.Fatalf("poison memory task: expected unsettled handle, got acks=%d nacks=%d", memHandle.acks.Load(), memHandle.nacks.Load())
}
if docHandle.acks.Load() != 1 || docHandle.nacks.Load() != 0 {
t.Fatalf("document task after memory panic: expected 1 Ack/0 Nack (worker survived), got acks=%d nacks=%d", docHandle.acks.Load(), docHandle.nacks.Load())
diff --git a/internal/ingestion/service/handle_and_execute_test.go b/internal/ingestion/service/handle_and_execute_test.go
index 4df6dd4944..9a7fbfbb5b 100644
--- a/internal/ingestion/service/handle_and_execute_test.go
+++ b/internal/ingestion/service/handle_and_execute_test.go
@@ -18,6 +18,7 @@ package service
import (
"context"
+ "errors"
"testing"
"time"
@@ -27,28 +28,30 @@ import (
servicepkg "ragflow/internal/service"
)
-// TestHandleAndExecute_MalformedMemoryPayloadAcks verifies that a memory task with
-// an unparseable payload is acked and skipped without executing the runner.
-func TestHandleAndExecute_MalformedMemoryPayloadAcks(t *testing.T) {
+// TestHandleAndExecute_MemoryTaskIDInvokesRunner verifies memory deliveries
+// need only the task id because execution input lives in the database.
+func TestHandleAndExecute_MemoryTaskIDInvokesRunner(t *testing.T) {
ingestor := newUnitIngestor("test-mem-malformed", 1, nil)
ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
runnerCalled := false
- ingestor.runMemoryTask = func(ctx context.Context, taskID string, payload map[string]any) error {
+ ingestor.runMemoryTask = func(ctx context.Context, taskID, leaseOwner string) (servicepkg.MemoryTaskDisposition, error) {
runnerCalled = true
- return nil
+ if taskID != "mem-bad-payload" || leaseOwner == "" {
+ t.Fatalf("runner identity = %q/%q", taskID, leaseOwner)
+ }
+ return servicepkg.MemoryTaskAcknowledge, nil
}
handle := &fakeTaskHandle{msg: common.TaskMessage{
TaskID: "mem-bad-payload",
TaskType: common.TaskTypeMemory,
- Payload: []byte(`{invalid-json`),
}}
ingestor.handleAndExecute(handle)
- if runnerCalled {
- t.Fatal("expected runMemoryTask to not be called for malformed payload")
+ if !runnerCalled {
+ t.Fatal("expected runMemoryTask to be called with task id only")
}
if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
t.Fatalf("expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
@@ -64,7 +67,6 @@ func TestHandleAndExecute_MemoryExtractorDisabledAcks(t *testing.T) {
handle := &fakeTaskHandle{msg: common.TaskMessage{
TaskID: "mem-disabled",
TaskType: common.TaskTypeMemory,
- Payload: []byte(`{"memory_id":"m1"}`),
}}
ingestor.handleAndExecute(handle)
@@ -83,7 +85,6 @@ func TestHandleAndExecute_MemoryEmptyTaskIDAcks(t *testing.T) {
handle := &fakeTaskHandle{msg: common.TaskMessage{
TaskID: "",
TaskType: common.TaskTypeMemory,
- Payload: []byte(`{"memory_id":"m1"}`),
}}
ingestor.handleAndExecute(handle)
@@ -277,33 +278,48 @@ func TestHandleAndExecute_DocumentDuplicateClaimRenewsLease(t *testing.T) {
}
}
-// TestHandleAndExecute_MemoryDuplicateClaimRenewsLease verifies duplicate delivery
-// protection for memory extraction tasks.
-func TestHandleAndExecute_MemoryDuplicateClaimRenewsLease(t *testing.T) {
+// TestHandleAndExecute_MemoryUnsettledDispositionDefersToDurableRecovery
+// verifies DB lease admission controls settlement for duplicate or failed work.
+func TestHandleAndExecute_MemoryUnsettledDispositionDefersToDurableRecovery(t *testing.T) {
ingestor := newUnitIngestor("test-mem-dup", 1, nil)
ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
-
- taskID := "mem-dup-1"
- if !ingestor.claimTask(taskID) {
- t.Fatal("first claim should succeed")
+ ingestor.runMemoryTask = func(context.Context, string, string) (servicepkg.MemoryTaskDisposition, error) {
+ return servicepkg.MemoryTaskLeaveUnsettled, nil
}
handle := &fakeTaskHandle{msg: common.TaskMessage{
- TaskID: taskID,
+ TaskID: "mem-dup-1",
TaskType: common.TaskTypeMemory,
- Payload: []byte(`{"memory_id":"m1","source_id":1,"message_dict":{"user_id":"u"}}`),
}}
ingestor.handleAndExecute(handle)
- if handle.inProgress.Load() != 1 {
- t.Fatalf("expected InProgress = 1 for duplicate memory delivery, got %d", handle.inProgress.Load())
- }
if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {
t.Fatalf("expected 0 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
}
}
+// TestHandleAndExecute_MemoryPersistedRetryAcks verifies an execution error is
+// still acknowledged after the service durably schedules its retry.
+func TestHandleAndExecute_MemoryPersistedRetryAcks(t *testing.T) {
+ ingestor := newUnitIngestor("test-mem-retry", 1, nil)
+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))
+ ingestor.runMemoryTask = func(context.Context, string, string) (servicepkg.MemoryTaskDisposition, error) {
+ return servicepkg.MemoryTaskAcknowledge, errors.New("retry scheduled")
+ }
+
+ handle := &fakeTaskHandle{msg: common.TaskMessage{
+ TaskID: "mem-retry-1",
+ TaskType: common.TaskTypeMemory,
+ }}
+
+ ingestor.handleAndExecute(handle)
+
+ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {
+ t.Fatalf("expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())
+ }
+}
+
// TestHandleAndExecute_SlowAdmissionHeartbeatsUnderLeaseProtection verifies that
// when admission (StartRunning) is slow (e.g. database lag exceeding ack deadline),
// the heartbeat started upon receiving the handle renews the broker lease before
diff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.go
index 4547454a4c..53e18fdad8 100644
--- a/internal/ingestion/service/ingestion_service.go
+++ b/internal/ingestion/service/ingestion_service.go
@@ -18,7 +18,6 @@ package service
import (
"context"
- "encoding/json"
"errors"
"fmt"
"runtime"
@@ -64,10 +63,12 @@ type Ingestor struct {
dispatchCancel context.CancelFunc
// Configuration
- maxConcurrency int32
- supportedDocTypes []string
- version string
- heartbeatInterval time.Duration
+ maxConcurrency int32
+ supportedDocTypes []string
+ version string
+ heartbeatInterval time.Duration
+ memoryReconcileInterval time.Duration
+ memoryReconcileBatchSize int
// Runtime state
currentTasks map[string]struct{} // set of task IDs currently claimed by a worker
@@ -111,7 +112,8 @@ type Ingestor struct {
kcEmbedding string
kcConcurrency int32 // number of parallel dataset-level compile workers
- compileWg sync.WaitGroup
+ compileWg sync.WaitGroup
+ memoryReconcileWg sync.WaitGroup
// runDocumentTask dispatches to the migrated task handler path.
// Tests may override this to verify branch routing without invoking
@@ -121,8 +123,13 @@ type Ingestor struct {
// runMemoryTask dispatches one async memory-extraction task. Tests may
// override this to inject a panicking/failing runner without a live DB or
// real MemoryMessageService. Defaults to defaultRunMemoryTask, which calls
- // memorySvc.HandleSaveToMemoryTask with the envelope task id.
- runMemoryTask func(ctx context.Context, taskID string, payload map[string]any) error
+ // memorySvc.HandleSaveToMemoryTask with the envelope task id and a unique
+ // database lease owner.
+ runMemoryTask func(ctx context.Context, taskID, leaseOwner string) (servicepkg.MemoryTaskDisposition, error)
+
+ // reconcileMemoryTasks republishes due durable memory task wake-ups. Tests
+ // may replace it to verify lifecycle behavior without a database or broker.
+ reconcileMemoryTasks func(ctx context.Context) error
// cancelCheck is polled periodically (every 3s) during task execution.
// When it returns true the task's context is cancelled, which causes the
@@ -151,25 +158,28 @@ func NewIngestor(name string, maxConcurrency int32, supportedTypes []string) *In
dispatchCtx, dispatchCancel := context.WithCancel(context.Background())
id := utility.GenerateUUID()
ingestor := &Ingestor{
- id: id,
- name: name,
- ctx: ctx,
- cancel: cancel,
- dispatchCtx: dispatchCtx,
- dispatchCancel: dispatchCancel,
- maxConcurrency: maxConcurrency,
- supportedDocTypes: supportedTypes,
- version: "1.0.0",
- currentTasks: make(map[string]struct{}),
- activeLeases: make(map[*Heartbeat]*activeLease),
- workerQueue: make(chan *worker, maxConcurrency),
- ShutdownCh: make(chan struct{}, 1),
- ingestionTaskSvc: servicepkg.NewIngestionTaskService(),
- docState: newDocStateUpdater(),
- heartbeatInterval: defaultHeartbeatInterval,
+ id: id,
+ name: name,
+ ctx: ctx,
+ cancel: cancel,
+ dispatchCtx: dispatchCtx,
+ dispatchCancel: dispatchCancel,
+ maxConcurrency: maxConcurrency,
+ supportedDocTypes: supportedTypes,
+ version: "1.0.0",
+ currentTasks: make(map[string]struct{}),
+ activeLeases: make(map[*Heartbeat]*activeLease),
+ workerQueue: make(chan *worker, maxConcurrency),
+ ShutdownCh: make(chan struct{}, 1),
+ ingestionTaskSvc: servicepkg.NewIngestionTaskService(),
+ docState: newDocStateUpdater(),
+ heartbeatInterval: defaultHeartbeatInterval,
+ memoryReconcileInterval: defaultMemoryTaskReconcileInterval,
+ memoryReconcileBatchSize: defaultMemoryTaskReconcileBatchSize,
}
ingestor.runDocumentTask = ingestor.defaultRunDocumentTask
ingestor.runMemoryTask = ingestor.defaultRunMemoryTask
+ ingestor.reconcileMemoryTasks = ingestor.defaultReconcileMemoryTasks
ingestor.cancelCheck = ingestor.defaultCancelCheck
ingestor.checkpointExists = canvas.RedisCheckpointExists
ingestor.kcConcurrency = maxConcurrency // parallel dataset-level compile workers default to the task width
@@ -187,6 +197,11 @@ const consumeErrorBackoff = 1 * time.Second
const taskPullRequestTimeout = 1 * time.Second
+const (
+ defaultMemoryTaskReconcileInterval = 10 * time.Second
+ defaultMemoryTaskReconcileBatchSize = 100
+)
+
func (e *Ingestor) Start() error {
common.Info(fmt.Sprintf("Ingestor %s initialized", e.id))
e.startOnce.Do(func() {
@@ -213,11 +228,12 @@ func (e *Ingestor) start() error {
}
}
- // Start the task worker pool and the dataset-level compile consumer as
- // owned goroutines joined by Stop via workerWg/compileWg. Start follows
+ // Start the task workers, memory reconciler, and dataset compile consumer as
+ // owned goroutines joined by Stop. Start follows
// the standard lifecycle contract: it returns immediately after kicking
// these off rather than blocking on the consume loop itself.
e.startWorkerPool()
+ e.startMemoryTaskReconciler()
e.startDatasetKnowledgeCompile()
// Run the main tasks.RAGFLOW dispatcher off the caller's goroutine so Start
@@ -228,6 +244,52 @@ func (e *Ingestor) start() error {
return nil
}
+// startMemoryTaskReconciler starts the owned loop that recovers durable memory
+// tasks whose initial wake-up was lost or whose retry became due.
+func (e *Ingestor) startMemoryTaskReconciler() {
+ if e.memorySvc == nil || dao.DB == nil || e.reconcileMemoryTasks == nil {
+ return
+ }
+ e.memoryReconcileWg.Add(1)
+ go e.memoryTaskReconcileLoop()
+}
+
+// memoryTaskReconcileLoop performs one startup pass and then polls until the
+// ingestor execution context is canceled.
+func (e *Ingestor) memoryTaskReconcileLoop() {
+ defer e.memoryReconcileWg.Done()
+ reconcile := func() {
+ if err := e.reconcileMemoryTasks(e.ctx); err != nil && !errors.Is(err, context.Canceled) {
+ common.Warn(fmt.Sprintf("reconcile durable memory tasks: %v", err))
+ }
+ }
+
+ reconcile()
+ interval := e.memoryReconcileInterval
+ if interval <= 0 {
+ interval = defaultMemoryTaskReconcileInterval
+ }
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-e.ctx.Done():
+ return
+ case <-ticker.C:
+ reconcile()
+ }
+ }
+}
+
+// defaultReconcileMemoryTasks publishes one bounded batch of due wake-ups.
+func (e *Ingestor) defaultReconcileMemoryTasks(ctx context.Context) error {
+ batchSize := e.memoryReconcileBatchSize
+ if batchSize <= 0 {
+ batchSize = defaultMemoryTaskReconcileBatchSize
+ }
+ return e.memorySvc.ReconcileMemoryTasks(ctx, batchSize)
+}
+
// consumeLoop is the worker dispatcher for tasks.RAGFLOW. It waits for an
// available worker before synchronously pulling one task, preventing prefetch
// and limiting this ingestor to one outstanding broker request.
@@ -412,18 +474,7 @@ func (e *Ingestor) handleAndExecute(handle common.TaskHandle) {
e.ackHandle(hb, handle, taskMessage.TaskID)
return
}
- var payload map[string]any
- if len(taskMessage.Payload) == 0 || json.Unmarshal(taskMessage.Payload, &payload) != nil {
- common.Warn(fmt.Sprintf("memory task %s has no parseable payload, ack", taskMessage.TaskID))
- e.ackHandle(hb, handle, taskMessage.TaskID)
- return
- }
- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(e.ctx, taskMessage.TaskID, payload, handle)
- if !e.claimTask(taskMessage.TaskID) {
- common.Warn(fmt.Sprintf("memory task %s redelivered while worker still processing, renew lease", taskMessage.TaskID))
- e.renewDuplicateHandle(hb, handle, taskMessage.TaskID)
- return
- }
+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(e.ctx, taskMessage.TaskID, handle)
e.executeMemoryTaskWithHeartbeat(e.ctx, taskCtx, hb)
return
}
@@ -589,30 +640,20 @@ func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskC
func (e *Ingestor) executeMemoryTaskWithHeartbeat(ctx context.Context, taskCtx *taskpkg.TaskContext, hb *Heartbeat) {
taskID := taskCtx.ID()
- var (
- settleAck bool
- settleNack bool
- )
+ settleAck := false
defer func() {
hb.Stop()
if r := recover(); r != nil {
common.Error(fmt.Sprintf("memory task %s panicked: %v", taskID, r), fmt.Errorf("%v", r))
- settleNack = true
}
if taskCtx.Handle == nil || e.leaseAbandoned(hb) {
- e.releaseTask(taskID)
return
}
if settleAck {
if err := taskCtx.Handle.Ack(); err != nil {
common.Error(fmt.Sprintf("ack memory task %s", taskID), err)
}
- } else if settleNack {
- if err := taskCtx.Handle.Nack(); err != nil {
- common.Error(fmt.Sprintf("nack memory task %s", taskID), err)
- }
}
- e.releaseTask(taskID)
}()
common.Info(fmt.Sprintf("Starting memory task %s", taskID))
@@ -625,29 +666,24 @@ func (e *Ingestor) executeMemoryTaskWithHeartbeat(ctx context.Context, taskCtx *
settleAck = true
return
}
- if err := e.runMemoryTask(ctx, taskID, taskCtx.MemoryPayload); err != nil {
- // defaultRunMemoryTask wraps terminal outcomes in ErrMemoryTaskTerminal
- // (durable progress=-1 written, completed progress>=1.0, or no row to
- // retry). Everything else is transient and must be redelivered rather
- // than dropped.
- if errors.Is(err, servicepkg.ErrMemoryTaskTerminal) {
- common.Error(fmt.Sprintf("memory task %s failed terminally, ack", taskID), err)
- settleAck = true
- return
- }
- common.Error(fmt.Sprintf("memory task %s failed transiently, nack for redelivery", taskID), err)
- settleNack = true
+ leaseOwner := fmt.Sprintf("%s:%s", e.id, utility.GenerateUUID())
+ disposition, err := e.runMemoryTask(ctx, taskID, leaseOwner)
+ if err != nil {
+ common.Error(fmt.Sprintf("memory task %s execution failed", taskID), err)
+ }
+ if disposition == servicepkg.MemoryTaskAcknowledge {
+ settleAck = true
+ common.Info(fmt.Sprintf("Memory task %s delivery acknowledged", taskID))
return
}
- common.Info(fmt.Sprintf("Memory task %s completed", taskID))
- settleAck = true
+ common.Warn(fmt.Sprintf("memory task %s delivery left unsettled for durable recovery", taskID))
}
// defaultRunMemoryTask is the production memory-task runner. It is held behind
// the runMemoryTask field so tests can substitute a panicking/failing runner
// without a live DB or real MemoryMessageService.
-func (e *Ingestor) defaultRunMemoryTask(ctx context.Context, taskID string, payload map[string]any) error {
- return e.memorySvc.HandleSaveToMemoryTask(ctx, taskID, payload)
+func (e *Ingestor) defaultRunMemoryTask(ctx context.Context, taskID, leaseOwner string) (servicepkg.MemoryTaskDisposition, error) {
+ return e.memorySvc.HandleSaveToMemoryTask(ctx, taskID, leaseOwner)
}
func (e *Ingestor) executeTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {
@@ -1177,6 +1213,7 @@ func (e *Ingestor) Stop(ctx context.Context) {
e.cancel()
e.workerWg.Wait()
e.compileWg.Wait()
+ e.memoryReconcileWg.Wait()
close(waitDone)
}()
diff --git a/internal/ingestion/service/ingestor_lifecycle_test.go b/internal/ingestion/service/ingestor_lifecycle_test.go
index 3bf1accb5c..b30c3c9c53 100644
--- a/internal/ingestion/service/ingestor_lifecycle_test.go
+++ b/internal/ingestion/service/ingestor_lifecycle_test.go
@@ -244,16 +244,15 @@ func TestStopDeadlineLeavesStuckMemoryHandleUnsettled(t *testing.T) {
release := make(chan struct{})
started := make(chan struct{})
- ingestor.runMemoryTask = func(context.Context, string, map[string]any) error {
+ ingestor.runMemoryTask = func(context.Context, string, string) (servicepkg.MemoryTaskDisposition, error) {
close(started)
<-release
- return nil
+ return servicepkg.MemoryTaskAcknowledge, nil
}
handle := &fakeTaskHandle{msg: common.TaskMessage{
TaskID: "memory-stop-timeout",
TaskType: common.TaskTypeMemory,
- Payload: []byte(`{}`),
}}
w := <-ingestor.workerQueue
w.inbox <- handle
@@ -474,6 +473,42 @@ func TestStartSchedulesCreatedTasks(t *testing.T) {
}
}
+// TestMemoryTaskReconcilerRunsAtStartupAndStopsWithIngestor verifies the
+// recovery loop is owned by the ingestor lifecycle.
+func TestMemoryTaskReconcilerRunsAtStartupAndStopsWithIngestor(t *testing.T) {
+ db := testutil.SetupTestDB(t)
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ ingestor := newUnitIngestor("test-memory-reconciler", 1, nil)
+ ingestor.memorySvc = &servicepkg.MemoryMessageService{}
+ ingestor.memoryReconcileInterval = time.Hour
+ called := make(chan struct{}, 1)
+ ingestor.reconcileMemoryTasks = func(context.Context) error {
+ called <- struct{}{}
+ return nil
+ }
+ ingestor.startMemoryTaskReconciler()
+
+ select {
+ case <-called:
+ case <-time.After(time.Second):
+ t.Fatal("memory task reconciler did not run its startup pass")
+ }
+
+ ingestor.cancel()
+ stopped := make(chan struct{})
+ go func() {
+ ingestor.memoryReconcileWg.Wait()
+ close(stopped)
+ }()
+ select {
+ case <-stopped:
+ case <-time.After(time.Second):
+ t.Fatal("memory task reconciler did not stop with the ingestor context")
+ }
+}
+
// TestWorkerDispatcherDoesNotActivateTaskUntilWorkerReceivesTask prevents a busy
// worker from prefetching its next task. task-2 must remain SCHEDULED while
// task-1 still occupies the only worker.
diff --git a/internal/ingestion/task/task_context.go b/internal/ingestion/task/task_context.go
index d41dfcb1ec..74eb4b8c74 100644
--- a/internal/ingestion/task/task_context.go
+++ b/internal/ingestion/task/task_context.go
@@ -32,16 +32,16 @@ type TaskKind int
const (
// TaskKindIngestion is an ingestion document task (IngestionTask set).
TaskKindIngestion TaskKind = iota
- // TaskKindMemory is an async memory-extraction task (MemoryPayload set,
- // IngestionTask nil). It shares the worker pool with ingestion tasks but
+ // TaskKindMemory is an async memory-extraction task (IngestionTask nil). It
+ // shares the worker pool with ingestion tasks but
// runs through executeMemoryTask instead of the ingestion state machine.
TaskKindMemory
)
// TaskContext holds the execution inputs for an ingestion document task or a
// memory-extraction task. Ingestion tasks populate IngestionTask and the
-// document/KB/tenant chain; memory tasks populate MemoryPayload and leave
-// IngestionTask nil.
+// document/KB/tenant chain; memory tasks carry only the durable task id and
+// leave IngestionTask nil.
type TaskContext struct {
Ctx context.Context
@@ -57,18 +57,10 @@ type TaskContext struct {
PipelineID string
File any
- // MemoryPayload carries the raw task_type="memory" message body for
- // memory tasks (memory_id/source_id/message_dict). Only set for
- // TaskKindMemory.
- MemoryPayload map[string]any
-
- // taskID is the envelope task identifier (TaskMessage.TaskID) that the
- // scheduler claims and later releases. It is the authoritative identity
- // for claim, release, and logging across both task kinds. Unexported so
- // the claim key cannot be mutated between admission (claim) and
- // settlement (release), which would leak the claim and permanently block
- // redelivery. There is deliberately no fallback chain in ID(): identity
- // lives only here, never re-derived from IngestionTask or MemoryPayload.
+ // taskID is the authoritative TaskMessage identity used for admission,
+ // execution, settlement, and logging. It is unexported so it cannot change
+ // while a worker owns the delivery. There is deliberately no fallback chain
+ // in ID(): identity lives only here, never in task-specific data.
taskID string
// Handle is the message-queue ack handle for the task message that scheduled
@@ -77,26 +69,21 @@ type TaskContext struct {
// - TaskKindIngestion: ack on a durably-persisted terminal status and
// nack otherwise (e.g. shutdown mid-task) so the message is redelivered
// and resumed after restart.
- // - TaskKindMemory: ack on success and on terminal failure (task absent,
- // already-failed, or progress=-1 persisted by HandleSaveToMemoryTask);
- // nack on transient failure (task-load DB error before any marker, or
- // LLM/network error that did not reach progress=-1) so the message is
- // redelivered. See executeMemoryTask.
+ // - TaskKindMemory: ack when the durable task runner permits it; otherwise
+ // leave the delivery unsettled for broker or reconciler recovery.
Handle common.TaskHandle
}
// NewMemoryTaskContextForScheduling creates a lightweight TaskContext for a
// memory-extraction task. taskID is the envelope TaskMessage.TaskID that the
-// scheduler claims and will release; it is the authoritative identity for the
-// whole memory task lifecycle. Only the scheduling-related fields are set, not
-// the full ingestion business data.
-func NewMemoryTaskContextForScheduling(ctx context.Context, taskID string, payload map[string]any, handle common.TaskHandle) *TaskContext {
+// scheduler receives; it is the authoritative identity for the whole memory
+// task lifecycle. Only scheduling fields are set, not full business data.
+func NewMemoryTaskContextForScheduling(ctx context.Context, taskID string, handle common.TaskHandle) *TaskContext {
return &TaskContext{
- Ctx: ctx,
- Kind: TaskKindMemory,
- taskID: taskID,
- MemoryPayload: payload,
- Handle: handle,
+ Ctx: ctx,
+ Kind: TaskKindMemory,
+ taskID: taskID,
+ Handle: handle,
}
}
@@ -113,7 +100,7 @@ func NewTaskContextForScheduling(ctx context.Context, task *entity.IngestionTask
// ID returns the task identifier for claim/release and logging. It is the
// envelope TaskMessage.TaskID captured at construction — there is deliberately
-// no fallback to IngestionTask or MemoryPayload, so identity is never
+// no fallback to IngestionTask or broker payload, so identity is never
// re-derived from a source that could disagree with the claim key.
func (c *TaskContext) ID() string {
if c == nil {
diff --git a/internal/service/memory.go b/internal/service/memory.go
index db9f9d29d5..07932b5cf9 100644
--- a/internal/service/memory.go
+++ b/internal/service/memory.go
@@ -1676,7 +1676,7 @@ func (s *MemoryService) GetMemoryConfig(ctx context.Context, userID, memoryID st
func (s *MemoryService) getMemoryConfig(ctx context.Context, memoryID string) (*CreateMemoryResponse, error) {
memory, err := s.memoryDAO.GetWithOwnerNameByID(ctx, dao.DB, memoryID)
if err != nil {
- return nil, fmt.Errorf("memory '%s' not found", memoryID)
+ return nil, fmt.Errorf("get memory %q: %w", memoryID, err)
}
return formatRetDataFromMemoryListItem(memory), nil
}
diff --git a/internal/service/memory_extractor.go b/internal/service/memory_extractor.go
index d6e5d8735f..4e07195848 100644
--- a/internal/service/memory_extractor.go
+++ b/internal/service/memory_extractor.go
@@ -39,6 +39,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "strconv"
"strings"
"time"
@@ -55,101 +56,350 @@ import (
// server-local wall-clock string ("YYYY-MM-DD HH:MM:SS"), never UTC-shifted.
const memoryTimeLayout = "2006-01-02 15:04:05"
+const (
+ memoryTaskLeaseTTL = 2 * time.Minute
+ memoryTaskRetryInitialDelay = 5 * time.Second
+ memoryTaskRetryMaxDelay = 5 * time.Minute
+ memoryTaskFailureWriteTimeout = 5 * time.Second
+ memoryTaskMaxAttempts = 10
+)
+
+var (
+ memoryTaskLeaseRenewInterval = 30 * time.Second
+ memoryTaskLeaseRenewTimeout = 10 * time.Second
+)
+
+var errPermanentMemoryTask = errors.New("memory: permanent task failure")
+
+// classifyMemoryTaskDependencyError marks confirmed missing or unusable
+// dependencies as permanent while preserving transient lookup failures.
+func classifyMemoryTaskDependencyError(err error) error {
+ if errors.Is(err, gorm.ErrRecordNotFound) || errors.Is(err, errModelConfigUnavailable) {
+ return fmt.Errorf("%w: %w", errPermanentMemoryTask, err)
+ }
+ return err
+}
+
// memoryNow is the wall clock behind every memory timestamp. Tests pin it
// to a fixed instant in a fixed location so the server-local assertions are
// deterministic on any host, including UTC CI runners.
var memoryNow = time.Now
-// ErrMemoryTaskTerminal marks a memory-task failure that already has a durable
-// terminal outcome (task row absent, or progress already persisted as failed),
-// so the caller must Ack rather than Nack/redeliver. Transient failures (DB
-// read hiccup, LLM/network errors before any durable marker) return plain
-// errors so executeMemoryTask can Nack and let the message be redelivered.
-var ErrMemoryTaskTerminal = errors.New("memory: terminal task failure, do not redeliver")
+// MemoryTaskDisposition tells the broker consumer whether a durable memory
+// task delivery can be acknowledged. An unsettled delivery is recovered by
+// broker redelivery or the database reconciler.
+type MemoryTaskDisposition uint8
-// errMemoryTaskRetryable marks a failure that must leave the broker message
-// unsettled. It is used when extracted messages may already be stored but the
-// durable task state could not be recorded; stable chunk ids make that retry
-// safe.
-var errMemoryTaskRetryable = errors.New("memory: retryable task failure")
+const (
+ // MemoryTaskLeaveUnsettled preserves the delivery for durable recovery.
+ MemoryTaskLeaveUnsettled MemoryTaskDisposition = iota
+ // MemoryTaskAcknowledge permits the consumer to settle the delivery.
+ MemoryTaskAcknowledge
+)
// extractedMemory is one LLM-extracted memory item ready for persistence.
type extractedMemory struct {
- MessageType string
- Content string
- ValidAt string
- InvalidAt string // empty means still valid
+ MessageID int64 `json:"message_id"`
+ MessageType string `json:"message_type"`
+ Content string `json:"content"`
+ ValidAt string `json:"valid_at"`
+ InvalidAt string `json:"invalid_at,omitempty"` // empty means still valid
}
-// HandleSaveToMemoryTask processes one queued memory task. Mirrors
-// Python handle_save_to_memory_task: validate the task row, then
-// extract + persist, settling task progress on the way out.
-//
-// taskID is the authoritative identity passed explicitly by the Ingestor (the
-// envelope TaskID); it is never re-derived from the payload. The payload
-// carries only business parameters (memory_id / source_id / message_dict).
-//
-// The returned error is wrapped in ErrMemoryTaskTerminal when the failure has
-// already produced a durable terminal outcome (dependency/config error, task
-// row absent, task already failed, or extraction failed after progress=-1 was
-// persisted). Transient failures (a task-load DB error before any marker was
-// written) return an unwrapped error so the caller can Nack and redeliver.
-func (s *MemoryMessageService) HandleSaveToMemoryTask(ctx context.Context, taskID string, payload map[string]any) error {
- if s == nil || s.taskDAO == nil || s.memories == nil {
- return fmt.Errorf("%w: memory: nil MemoryMessageService or memory dependency", ErrMemoryTaskTerminal)
- }
- memoryID, _ := payload["memory_id"].(string)
- sourceID := payloadInt64(payload["source_id"])
- msgDict, _ := payload["message_dict"].(map[string]any)
- msg := MemoryMessage{
- UserID: payloadString(msgDict["user_id"]),
- AgentID: payloadString(msgDict["agent_id"]),
- SessionID: payloadString(msgDict["session_id"]),
- UserInput: payloadString(msgDict["user_input"]),
- AgentResponse: payloadString(msgDict["agent_response"]),
+type memoryExtractionCheckpoint struct {
+ MessageID string `json:"message_id"`
+ MessageType string `json:"message_type"`
+ Content string `json:"content"`
+ ValidAt string `json:"valid_at"`
+ InvalidAt string `json:"invalid_at,omitempty"`
+}
+
+// HandleSaveToMemoryTask claims and resumes one durable memory task. The NATS
+// delivery is only a wake-up; task input and checkpoints always come from the
+// memory_task row identified by taskID.
+func (s *MemoryMessageService) HandleSaveToMemoryTask(ctx context.Context, taskID, leaseOwner string) (MemoryTaskDisposition, error) {
+ if s == nil {
+ return MemoryTaskAcknowledge, errors.New("memory: nil MemoryMessageService")
+ }
+ if taskID == "" || leaseOwner == "" {
+ return MemoryTaskLeaveUnsettled, errors.New("memory: task id and lease owner are required")
+ }
+ if s.memoryTaskDAO == nil {
+ s.memoryTaskDAO = dao.NewMemoryTaskDAO()
+ }
+
+ task, acquired, err := s.memoryTaskDAO.Claim(ctx, dao.DB, taskID, leaseOwner, memoryNow(), memoryTaskLeaseTTL)
+ if err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return s.failTaskMissingDurableState(ctx, taskID)
+ }
+ return MemoryTaskLeaveUnsettled, fmt.Errorf("memory: claim task %s: %w", taskID, err)
+ }
+ if !acquired {
+ switch task.State {
+ case entity.MemoryTaskStateCompleted, entity.MemoryTaskStateFailed:
+ return MemoryTaskAcknowledge, nil
+ case entity.MemoryTaskStatePending, entity.MemoryTaskStateExtracted, entity.MemoryTaskStateStored:
+ return MemoryTaskLeaveUnsettled, nil
+ default:
+ return MemoryTaskAcknowledge, fmt.Errorf("memory: task %s has unknown state %q", taskID, task.State)
+ }
+ }
+
+ return s.runClaimedMemoryTask(ctx, task, leaseOwner)
+}
+
+// runClaimedMemoryTask renews the DB lease while the state machine advances.
+func (s *MemoryMessageService) runClaimedMemoryTask(ctx context.Context, task *entity.MemoryTask, leaseOwner string) (MemoryTaskDisposition, error) {
+ if task.LeaseExpiresAt == nil {
+ return MemoryTaskLeaveUnsettled, fmt.Errorf("memory: claimed task %s has no lease expiration", task.TaskID)
+ }
+ leaseExpiresAt := *task.LeaseExpiresAt
+ runCtx, cancel := context.WithCancel(ctx)
+ renewErr := make(chan error, 1)
+ renewDone := make(chan struct{})
+ go func() {
+ defer close(renewDone)
+ s.renewMemoryTaskLease(runCtx, cancel, task.TaskID, leaseOwner, leaseExpiresAt, renewErr)
+ }()
+ defer func() {
+ cancel()
+ <-renewDone
+ }()
+
+ resumeTask := s.resumeMemoryTask
+ if s.resumeTask != nil {
+ resumeTask = s.resumeTask
+ }
+ err := func() (err error) {
+ defer func() {
+ if recovered := recover(); recovered != nil {
+ err = fmt.Errorf("memory: task %s panicked: %v", task.TaskID, recovered)
+ }
+ }()
+ return resumeTask(runCtx, task, leaseOwner)
+ }()
+ if err == nil {
+ return MemoryTaskAcknowledge, nil
+ }
+ select {
+ case leaseErr := <-renewErr:
+ if leaseErr != nil {
+ err = leaseErr
+ }
+ default:
+ }
+ return s.persistMemoryTaskFailure(ctx, task, leaseOwner, err)
+}
+
+// persistMemoryTaskFailure records the recovery decision before allowing the
+// broker delivery to be acknowledged. If the write cannot be proven, the
+// delivery remains unsettled and the expired lease is recovered later.
+func (s *MemoryMessageService) persistMemoryTaskFailure(ctx context.Context, task *entity.MemoryTask, leaseOwner string, runErr error) (MemoryTaskDisposition, error) {
+ persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), memoryTaskFailureWriteTimeout)
+ defer cancel()
+
+ var (
+ updated bool
+ err error
+ action string
+ )
+ now := memoryNow()
+ reportedErr := runErr
+ if !errors.Is(runErr, errPermanentMemoryTask) && task.AttemptCount >= memoryTaskMaxAttempts {
+ reportedErr = fmt.Errorf("memory: task failed after %d attempts: %w", task.AttemptCount, runErr)
+ }
+ if errors.Is(runErr, errPermanentMemoryTask) || task.AttemptCount >= memoryTaskMaxAttempts {
+ action = "mark failed"
+ updated, err = s.memoryTaskDAO.MarkFailed(persistCtx, dao.DB, task.TaskID, leaseOwner, reportedErr.Error(), now)
+ } else {
+ action = "schedule retry"
+ nextRetryAt := now.Add(memoryTaskRetryDelay(task.AttemptCount))
+ updated, err = s.memoryTaskDAO.ScheduleRetry(persistCtx, dao.DB, task.TaskID, leaseOwner, now, nextRetryAt, reportedErr.Error())
+ }
+ if err != nil {
+ return MemoryTaskLeaveUnsettled, errors.Join(reportedErr, fmt.Errorf("memory: %s for task %s: %w", action, task.TaskID, err))
+ }
+ if !updated {
+ return MemoryTaskLeaveUnsettled, errors.Join(reportedErr, fmt.Errorf("memory: %s for task %s: lease is no longer owned by this worker", action, task.TaskID))
}
+ return MemoryTaskAcknowledge, reportedErr
+}
+// failTaskMissingDurableState makes an unfinished task visible as failed
+// instead of silently acknowledging a delivery that cannot resume.
+func (s *MemoryMessageService) failTaskMissingDurableState(ctx context.Context, taskID string) (MemoryTaskDisposition, error) {
+ if s.taskDAO == nil {
+ s.taskDAO = dao.NewTaskDAO()
+ }
task, err := s.taskDAO.GetByID(ctx, dao.DB, taskID)
if err != nil {
- // Record-not-found is terminal: no row to retry against. Any other
- // task-load error is transient and must be redelivered (no progress=-1
- // marker was written).
if errors.Is(err, gorm.ErrRecordNotFound) {
- return fmt.Errorf("%w: memory: task %s is not found", ErrMemoryTaskTerminal, taskID)
+ return MemoryTaskAcknowledge, fmt.Errorf("memory: task %s is not found", taskID)
}
- return fmt.Errorf("memory: load task %s: %w", taskID, err)
+ return MemoryTaskLeaveUnsettled, fmt.Errorf("memory: load task %s after durable state lookup: %w", taskID, err)
}
- if task.Progress == -1 {
- return fmt.Errorf("%w: memory: task %s is already failed", ErrMemoryTaskTerminal, taskID)
+ if task.TaskType != common.TaskTypeMemory || task.Progress < 0 || task.Progress >= 1 {
+ return MemoryTaskAcknowledge, fmt.Errorf("memory: task %s has no active durable execution state", taskID)
+ }
+
+ failure := fmt.Errorf("memory: task %s cannot resume because it is missing durable execution state after the server upgrade; save the message again", taskID)
+ persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), memoryTaskFailureWriteTimeout)
+ defer cancel()
+ updated, updateErr := s.taskDAO.MarkActiveMemoryTaskFailed(persistCtx, dao.DB, taskID, failure.Error())
+ if updateErr != nil {
+ return MemoryTaskLeaveUnsettled, errors.Join(failure, fmt.Errorf("memory: mark task failed: %w", updateErr))
+ }
+ if !updated {
+ return MemoryTaskAcknowledge, fmt.Errorf("%w: task is no longer active", failure)
+ }
+ common.Warn(failure.Error())
+ return MemoryTaskAcknowledge, failure
+}
+
+// memoryTaskRetryDelay applies bounded exponential backoff using the attempt
+// count incremented when the task lease was claimed.
+func memoryTaskRetryDelay(attemptCount int) time.Duration {
+ delay := memoryTaskRetryInitialDelay
+ for attempt := 1; attempt < attemptCount && delay < memoryTaskRetryMaxDelay; attempt++ {
+ delay *= 2
+ if delay >= memoryTaskRetryMaxDelay {
+ return memoryTaskRetryMaxDelay
+ }
}
- // A task already extracted to completion (progress>=1.0) is a durable
- // terminal outcome: a redelivery after restart (or a duplicate copy from
- // another consumer) must not re-run the LLM extraction, which would insert
- // duplicate memory entries. Short-circuit to success so the Ingestor Acks
- // the message instead of re-executing the task.
- if task.Progress >= 1.0 {
+ return delay
+}
+
+// renewMemoryTaskLease retries transient renewal failures while the last
+// confirmed lease has time for another attempt, and cancels execution when
+// ownership is lost or the lease is too close to expiry.
+func (s *MemoryMessageService) renewMemoryTaskLease(ctx context.Context, cancel context.CancelFunc, taskID, leaseOwner string, leaseExpiresAt time.Time, errCh chan<- error) {
+ ticker := time.NewTicker(memoryTaskLeaseRenewInterval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ renewedAt := memoryNow()
+ renewCtx, renewCancel := context.WithTimeout(ctx, memoryTaskLeaseRenewTimeout)
+ renewed, err := s.memoryTaskDAO.RenewLease(renewCtx, dao.DB, taskID, leaseOwner, renewedAt, memoryTaskLeaseTTL)
+ renewCancel()
+ if err == nil && renewed {
+ leaseExpiresAt = renewedAt.Add(memoryTaskLeaseTTL)
+ continue
+ }
+ if err != nil && memoryNow().Add(memoryTaskLeaseRenewInterval).Before(leaseExpiresAt) {
+ common.Warn(fmt.Sprintf("memory: renew task %s lease failed, will retry", taskID), zap.Error(err))
+ continue
+ }
+ if err == nil {
+ err = errors.New("lease is no longer owned by this worker")
+ }
+ select {
+ case errCh <- fmt.Errorf("memory: renew task %s lease: %w", taskID, err):
+ default:
+ }
+ cancel()
+ return
+ }
+ }
+}
+
+// resumeMemoryTask advances from the last durable checkpoint through
+// completion without consulting the generic task progress projection.
+func (s *MemoryMessageService) resumeMemoryTask(ctx context.Context, task *entity.MemoryTask, leaseOwner string) error {
+ var (
+ msg MemoryMessage
+ inputLoaded bool
+ )
+ loadInput := func() error {
+ if inputLoaded {
+ return nil
+ }
+ var err error
+ msg, err = memoryMessageFromTaskInput(task.Input)
+ if err != nil {
+ return fmt.Errorf("%w: decode task %s input: %v", errPermanentMemoryTask, task.TaskID, err)
+ }
+ inputLoaded = true
return nil
}
- if err = s.saveExtractedToMemory(ctx, memoryID, msg, sourceID, taskID); err != nil {
- if errors.Is(err, errMemoryTaskRetryable) {
+ for {
+ if err := ctx.Err(); err != nil {
return err
}
- if progressErr := s.updateTaskProgress(ctx, taskID, -1, err.Error()); progressErr != nil {
- return fmt.Errorf("%w: memory: mark task %s failed: %v", errMemoryTaskRetryable, taskID, progressErr)
+ switch task.State {
+ case entity.MemoryTaskStatePending:
+ if err := loadInput(); err != nil {
+ return err
+ }
+ extraction, extractErr := s.extractMemoryTask(ctx, task, msg)
+ if extractErr != nil {
+ return extractErr
+ }
+ encoded, encodeErr := encodeMemoryExtraction(extraction)
+ if encodeErr != nil {
+ return fmt.Errorf("memory: encode task %s extraction: %w", task.TaskID, encodeErr)
+ }
+ updated, persistErr := s.memoryTaskDAO.PersistExtraction(ctx, dao.DB, task.TaskID, leaseOwner, memoryNow(), encoded)
+ if persistErr != nil {
+ return fmt.Errorf("memory: persist task %s extraction: %w", task.TaskID, persistErr)
+ }
+ if !updated {
+ return fmt.Errorf("memory: task %s lost its lease before extraction checkpoint", task.TaskID)
+ }
+ task.Extraction = encoded
+ task.State = entity.MemoryTaskStateExtracted
+ _ = s.updateTaskProgress(ctx, task.TaskID, 0.5, fmt.Sprintf("Extracted %d messages from raw dialogue.", len(extraction)))
+
+ case entity.MemoryTaskStateExtracted:
+ if err := loadInput(); err != nil {
+ return err
+ }
+ if err := s.storeMemoryTaskExtraction(ctx, task, msg); err != nil {
+ return err
+ }
+ updated, storeErr := s.memoryTaskDAO.MarkStored(ctx, dao.DB, task.TaskID, leaseOwner, memoryNow())
+ if storeErr != nil {
+ return fmt.Errorf("memory: mark task %s stored: %w", task.TaskID, storeErr)
+ }
+ if !updated {
+ return fmt.Errorf("memory: task %s lost its lease before storage checkpoint", task.TaskID)
+ }
+ task.State = entity.MemoryTaskStateStored
+
+ case entity.MemoryTaskStateStored:
+ progressMsg := "Message saved successfully."
+ if len(task.Extraction) == 0 {
+ progressMsg = "No memory extracted from raw message."
+ }
+ completed, completeErr := s.memoryTaskDAO.Complete(ctx, dao.DB, task.TaskID, leaseOwner, progressMsg, memoryNow())
+ if completeErr != nil {
+ return fmt.Errorf("memory: complete task %s: %w", task.TaskID, completeErr)
+ }
+ if !completed {
+ return fmt.Errorf("memory: task %s lost its lease before completion", task.TaskID)
+ }
+ return nil
+
+ case entity.MemoryTaskStateCompleted, entity.MemoryTaskStateFailed:
+ return nil
+ default:
+ return fmt.Errorf("%w: task %s has unknown state %q", errPermanentMemoryTask, task.TaskID, task.State)
}
- return fmt.Errorf("%w: %w", ErrMemoryTaskTerminal, err)
}
- return nil
}
-// saveExtractedToMemory mirrors Python save_extracted_to_memory_only:
-// skip raw-only memories, run LLM extraction, embed and persist the
-// extracted messages under the raw message's id.
-func (s *MemoryMessageService) saveExtractedToMemory(ctx context.Context, memoryID string, msg MemoryMessage, sourceID int64, taskID string) error {
- mem, err := s.memories.getMemoryConfig(ctx, memoryID)
+// extractMemoryTask executes the LLM stage and materializes retry-stable output.
+func (s *MemoryMessageService) extractMemoryTask(ctx context.Context, task *entity.MemoryTask, msg MemoryMessage) ([]extractedMemory, error) {
+ if s.memories == nil {
+ return nil, errors.New("memory: memory service is not initialized")
+ }
+ mem, err := s.memories.getMemoryConfig(ctx, task.MemoryID)
if err != nil {
- return err
+ return nil, classifyMemoryTaskDependencyError(err)
}
memoryTypes := mem.MemoryType
if len(memoryTypes) == 0 {
@@ -157,35 +407,39 @@ func (s *MemoryMessageService) saveExtractedToMemory(ctx context.Context, memory
}
extractTypes := getTypesToExtract(memoryTypes)
if len(extractTypes) == 0 {
- if err := s.updateTaskProgress(ctx, taskID, 1.0, fmt.Sprintf("Memory '%s' don't need to extract.", memoryID)); err != nil {
- return fmt.Errorf("%w: memory: mark task %s complete: %w", errMemoryTaskRetryable, taskID, err)
- }
- return nil
+ return []extractedMemory{}, nil
}
- extracted, err := s.extractByLLM(ctx, mem, extractTypes, msg, taskID)
+ extracted, err := s.extractByLLM(ctx, mem, extractTypes, msg, task.TaskID)
if err != nil {
- return err
+ return nil, err
+ }
+ materialized := materializeMemoryExtraction(ctx, extracted, memoryNow())
+ return materialized, nil
+}
+
+// storeMemoryTaskExtraction embeds and stores a previously checkpointed result.
+func (s *MemoryMessageService) storeMemoryTaskExtraction(ctx context.Context, task *entity.MemoryTask, msg MemoryMessage) error {
+ extracted, err := decodeMemoryExtraction(task.Extraction)
+ if err != nil {
+ return fmt.Errorf("%w: decode task %s extraction: %v", errPermanentMemoryTask, task.TaskID, err)
}
if len(extracted) == 0 {
- if err := s.updateTaskProgress(ctx, taskID, 1.0, "No memory extracted from raw message."); err != nil {
- return fmt.Errorf("%w: memory: mark task %s complete: %w", errMemoryTaskRetryable, taskID, err)
- }
return nil
}
- _ = s.updateTaskProgress(ctx, taskID, 0.5, fmt.Sprintf("Extracted %d messages from raw dialogue.", len(extracted)))
-
- // conversation_time is stamped as server-local wall clock, not UTC.
- now := memoryNow()
+ if s.memories == nil {
+ return errors.New("memory: memory service is not initialized")
+ }
+ mem, err := s.memories.getMemoryConfig(ctx, task.MemoryID)
+ if err != nil {
+ return classifyMemoryTaskDependencyError(err)
+ }
messages := make([]map[string]any, 0, len(extracted))
for _, item := range extracted {
- messages = append(messages, buildExtractedMessage(generateRawMessageID(ctx), sourceID, memoryID, msg, item, now))
+ messages = append(messages, buildExtractedMessage(task.SourceID, task.MemoryID, msg, item))
}
if err = s.embedAndSaveMessages(ctx, mem, messages); err != nil {
- return err
- }
- if err := s.updateTaskProgress(ctx, taskID, 1.0, "Message saved successfully."); err != nil {
- return fmt.Errorf("%w: memory: mark task %s complete: %w", errMemoryTaskRetryable, taskID, err)
+ return classifyMemoryTaskDependencyError(err)
}
return nil
}
@@ -222,7 +476,7 @@ func (s *MemoryMessageService) extractByLLM(ctx context.Context, mem *CreateMemo
}
driver, modelName, apiConfig, _, err := NewModelProviderService().ResolveModelConfig(ctx, mem.TenantID, entity.ModelTypeChat, llmRef)
if err != nil {
- return nil, fmt.Errorf("resolve chat model: %w", err)
+ return nil, fmt.Errorf("resolve chat model: %w", classifyMemoryTaskDependencyError(err))
}
chatModel := models.NewChatModel(driver, &modelName, apiConfig)
@@ -235,32 +489,108 @@ func (s *MemoryMessageService) extractByLLM(ctx context.Context, mem *CreateMemo
if resp == nil || resp.Answer == nil {
return nil, errors.New("empty response from chat model")
}
- _ = s.updateTaskProgress(ctx, taskID, 0.35, "Get extracted result from LLM.")
return parseMemoryExtraction(*resp.Answer, extractTypes), nil
}
-// buildExtractedMessage builds the persisted envelope for one extracted
-// memory item. Field set matches buildRawMessage except message_type and
-// source_id, which listMemoryMessages uses to aggregate extracts. Only
-// logical message fields are set here; the doc engine maps them to
-// storage fields (including tokenization) at insert time.
-func buildExtractedMessage(messageID, sourceID int64, memoryID string, msg MemoryMessage, item extractedMemory, now time.Time) map[string]any {
- validAt, ok := normalizeMemoryTime(item.ValidAt)
- if !ok {
- validAt = now.Format(memoryTimeLayout)
- }
- invalidAt, ok := normalizeMemoryTime(item.InvalidAt)
- if strings.TrimSpace(item.InvalidAt) != "" && !ok {
- invalidAt = now.Format(memoryTimeLayout)
+// materializeMemoryExtraction assigns stable message ids and timestamp
+// fallbacks before the extraction checkpoint is persisted.
+func materializeMemoryExtraction(ctx context.Context, extracted []extractedMemory, now time.Time) []extractedMemory {
+ materialized := make([]extractedMemory, len(extracted))
+ for i, item := range extracted {
+ item.MessageID = generateRawMessageID(ctx)
+ item.ValidAt = formatMemoryTime(item.ValidAt, now)
+ item.InvalidAt = strings.TrimSpace(item.InvalidAt)
+ if item.InvalidAt != "" {
+ item.InvalidAt = formatMemoryTime(item.InvalidAt, now)
+ }
+ materialized[i] = item
+ }
+ return materialized
+}
+
+// encodeMemoryExtraction converts typed extraction output to the JSON column type.
+func encodeMemoryExtraction(extracted []extractedMemory) (entity.JSONSlice, error) {
+ checkpoint := make([]memoryExtractionCheckpoint, len(extracted))
+ for i, item := range extracted {
+ checkpoint[i] = memoryExtractionCheckpoint{
+ MessageID: strconv.FormatInt(item.MessageID, 10),
+ MessageType: item.MessageType,
+ Content: item.Content,
+ ValidAt: item.ValidAt,
+ InvalidAt: item.InvalidAt,
+ }
+ }
+ data, err := json.Marshal(checkpoint)
+ if err != nil {
+ return nil, err
}
+ var encoded entity.JSONSlice
+ if err = json.Unmarshal(data, &encoded); err != nil {
+ return nil, err
+ }
+ return encoded, nil
+}
+
+// decodeMemoryExtraction restores typed extraction output from its checkpoint.
+func decodeMemoryExtraction(encoded entity.JSONSlice) ([]extractedMemory, error) {
+ data, err := json.Marshal(encoded)
+ if err != nil {
+ return nil, err
+ }
+ var checkpoint []memoryExtractionCheckpoint
+ if err = json.Unmarshal(data, &checkpoint); err != nil {
+ return nil, err
+ }
+ extracted := make([]extractedMemory, len(checkpoint))
+ for i, item := range checkpoint {
+ messageID, parseErr := strconv.ParseInt(item.MessageID, 10, 64)
+ if parseErr != nil {
+ return nil, fmt.Errorf("parse message id %q: %w", item.MessageID, parseErr)
+ }
+ extracted[i] = extractedMemory{
+ MessageID: messageID,
+ MessageType: item.MessageType,
+ Content: item.Content,
+ ValidAt: item.ValidAt,
+ InvalidAt: item.InvalidAt,
+ }
+ }
+ return extracted, nil
+}
+
+// memoryMessageFromTaskInput reads the dialogue persisted with the durable task.
+func memoryMessageFromTaskInput(input entity.JSONMap) (MemoryMessage, error) {
+ msg := MemoryMessage{
+ UserID: stringFromJSONMap(input, "user_id"),
+ AgentID: stringFromJSONMap(input, "agent_id"),
+ SessionID: stringFromJSONMap(input, "session_id"),
+ UserInput: stringFromJSONMap(input, "user_input"),
+ AgentResponse: stringFromJSONMap(input, "agent_response"),
+ }
+ if msg.AgentID == "" {
+ return MemoryMessage{}, errors.New("agent_id is required")
+ }
+ return msg, nil
+}
+
+// stringFromJSONMap returns the named string value or an empty string.
+func stringFromJSONMap(values entity.JSONMap, key string) string {
+ value, _ := values[key].(string)
+ return value
+}
+
+// buildExtractedMessage builds the persisted envelope for one materialized
+// memory item. Field set matches buildRawMessage except message_type and
+// source_id, which listMemoryMessages uses to aggregate extracts.
+func buildExtractedMessage(sourceID int64, memoryID string, msg MemoryMessage, item extractedMemory) map[string]any {
var storedInvalidAt any
- if invalidAt != "" {
- storedInvalidAt = invalidAt
+ if item.InvalidAt != "" {
+ storedInvalidAt = item.InvalidAt
}
return map[string]any{
- "id": fmt.Sprintf("%s_%d", memoryID, messageID),
- "message_id": messageID,
+ "id": fmt.Sprintf("%s_%d", memoryID, item.MessageID),
+ "message_id": item.MessageID,
"message_type": item.MessageType,
"source_id": sourceID,
"memory_id": memoryID,
@@ -268,7 +598,7 @@ func buildExtractedMessage(messageID, sourceID int64, memoryID string, msg Memor
"agent_id": msg.AgentID,
"session_id": msg.SessionID,
"content": item.Content,
- "valid_at": validAt,
+ "valid_at": item.ValidAt,
"invalid_at": storedInvalidAt,
"forget_at": nil,
"status": true,
@@ -343,9 +673,8 @@ func normalizeMemoryTime(value string) (string, bool) {
return "", false
}
-// updateTaskProgress stamps and persists task progress, mirroring
-// Python TaskService.update_progress call sites. Callers that establish a
-// terminal task state must handle an error so the message can be retried.
+// updateTaskProgress stamps and persists the UI progress projection. Durable
+// execution decisions never read this value.
func (s *MemoryMessageService) updateTaskProgress(ctx context.Context, taskID string, progress float64, msg string) error {
if s == nil || s.taskDAO == nil {
return errors.New("memory: nil task DAO")
@@ -360,27 +689,3 @@ func (s *MemoryMessageService) updateTaskProgress(ctx context.Context, taskID st
}
return nil
}
-
-func payloadInt64(v any) int64 {
- switch n := v.(type) {
- case float64:
- return int64(n)
- case int64:
- return n
- case int:
- return int64(n)
- case json.Number:
- i, _ := n.Int64()
- return i
- case string:
- var i int64
- fmt.Sscanf(n, "%d", &i)
- return i
- }
- return 0
-}
-
-func payloadString(v any) string {
- s, _ := v.(string)
- return s
-}
diff --git a/internal/service/memory_extractor_test.go b/internal/service/memory_extractor_test.go
index fd305ffc1e..ea2c418e23 100644
--- a/internal/service/memory_extractor_test.go
+++ b/internal/service/memory_extractor_test.go
@@ -20,13 +20,19 @@
package service
import (
+ "context"
+ "errors"
"strings"
+ "sync/atomic"
"testing"
"time"
+ "ragflow/internal/common"
"ragflow/internal/dao"
"ragflow/internal/entity"
"ragflow/internal/ingestion/testutil"
+
+ "gorm.io/gorm"
)
// pinMemoryNow fixes the memory wall clock at the given instant for the
@@ -38,6 +44,20 @@ func pinMemoryNow(t *testing.T, instant time.Time) {
t.Cleanup(func() { memoryNow = orig })
}
+// pinMemoryTaskLeaseTimings shortens lease maintenance for focused lifecycle
+// tests and restores the production values afterwards.
+func pinMemoryTaskLeaseTimings(t *testing.T, interval, timeout time.Duration) {
+ t.Helper()
+ originalInterval := memoryTaskLeaseRenewInterval
+ originalTimeout := memoryTaskLeaseRenewTimeout
+ memoryTaskLeaseRenewInterval = interval
+ memoryTaskLeaseRenewTimeout = timeout
+ t.Cleanup(func() {
+ memoryTaskLeaseRenewInterval = originalInterval
+ memoryTaskLeaseRenewTimeout = originalTimeout
+ })
+}
+
// TestFormatMemoryTimeKeepsParsedWallClock: an offset-bearing or
// Z-suffixed ISO timestamp must keep its own wall clock — never be
// converted to UTC the way time.Time.UTC would.
@@ -81,22 +101,34 @@ func TestBuildExtractedMessageValidAtSemantics(t *testing.T) {
msg := MemoryMessage{UserID: "u1", AgentID: "a1", SessionID: "s1"}
now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.Local)
- withLLMTime := buildExtractedMessage(8, 42, "mem-1", msg, extractedMemory{
+ materialized := materializeMemoryExtraction(t.Context(), []extractedMemory{{
MessageType: "fact",
Content: "likes coffee",
ValidAt: "2026-08-20T10:05:00+08:00",
- }, now)
+ }}, now)
+ withLLMTime := buildExtractedMessage(42, "mem-1", msg, materialized[0])
if got := withLLMTime["valid_at"]; got != "2026-08-20 10:05:00" {
t.Fatalf("valid_at = %v, want LLM wall clock %q", got, "2026-08-20 10:05:00")
}
- fallbackOnly := buildExtractedMessage(9, 42, "mem-1", msg, extractedMemory{
+ materialized = materializeMemoryExtraction(t.Context(), []extractedMemory{{
MessageType: "fact",
Content: "likes coffee",
- }, now)
+ }}, now)
+ fallbackOnly := buildExtractedMessage(42, "mem-1", msg, materialized[0])
if got := fallbackOnly["valid_at"]; got != now.Format(memoryTimeLayout) {
t.Fatalf("valid_at = %v, want fallback %q", got, now.Format(memoryTimeLayout))
}
+
+ materialized = materializeMemoryExtraction(t.Context(), []extractedMemory{{
+ MessageType: "fact",
+ Content: "likes coffee",
+ InvalidAt: " ",
+ }}, now)
+ blankInvalidAt := buildExtractedMessage(42, "mem-1", msg, materialized[0])
+ if got := blankInvalidAt["invalid_at"]; got != nil {
+ t.Fatalf("invalid_at = %#v, want nil for whitespace-only input", got)
+ }
}
// TestBuildExtractedMessageUsesStandardDocumentID ensures extracted items
@@ -105,19 +137,822 @@ func TestBuildExtractedMessageUsesStandardDocumentID(t *testing.T) {
msg := MemoryMessage{UserID: "u1", AgentID: "a1", SessionID: "s1"}
now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.Local)
item := extractedMemory{
+ MessageID: 8,
MessageType: "fact",
Content: "likes coffee",
+ ValidAt: now.Format(memoryTimeLayout),
}
- extracted := buildExtractedMessage(8, 42, "mem-1", msg, item, now)
+ extracted := buildExtractedMessage(42, "mem-1", msg, item)
gotID, ok := extracted["id"].(string)
if !ok || gotID != "mem-1_8" {
t.Fatalf("extracted message id = %#v, want %q", extracted["id"], "mem-1_8")
}
}
-// TestUpdateTaskProgressReturnsPersistenceError ensures callers can keep the
-// broker message retryable when they cannot persist the terminal task state.
+// TestMemoryExtractionCheckpointRoundTripPreservesMaterializedFields verifies
+// retries reuse the same message id and timestamps.
+func TestMemoryExtractionCheckpointRoundTripPreservesMaterializedFields(t *testing.T) {
+ want := []extractedMemory{{
+ MessageID: 1700000000000000123,
+ MessageType: "fact",
+ Content: "likes coffee",
+ ValidAt: "2026-08-20 10:05:00",
+ }}
+ encoded, err := encodeMemoryExtraction(want)
+ if err != nil {
+ t.Fatalf("encodeMemoryExtraction: %v", err)
+ }
+ got, err := decodeMemoryExtraction(encoded)
+ if err != nil {
+ t.Fatalf("decodeMemoryExtraction: %v", err)
+ }
+ if len(got) != 1 || got[0] != want[0] {
+ t.Fatalf("checkpoint round trip = %+v, want %+v", got, want)
+ }
+}
+
+// TestExtractMemoryTaskMarksMissingMemoryPermanent verifies a deleted memory
+// terminates its durable task instead of entering the retry schedule forever.
+func TestExtractMemoryTaskMarksMissingMemoryPermanent(t *testing.T) {
+ db := testutil.SetupTestDB(t, &entity.Memory{}, &entity.User{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ svc := NewMemoryMessageService(NewMemoryService())
+ _, err := svc.extractMemoryTask(t.Context(), &entity.MemoryTask{MemoryID: "missing-memory"}, MemoryMessage{})
+ if !errors.Is(err, errPermanentMemoryTask) || !errors.Is(err, gorm.ErrRecordNotFound) {
+ t.Fatalf("extractMemoryTask error = %v, want permanent missing-memory error", err)
+ }
+}
+
+// TestExtractMemoryTaskKeepsLookupFailureRetryable verifies an unexpected
+// database failure is not mistaken for a deleted dependency.
+func TestExtractMemoryTaskKeepsLookupFailureRetryable(t *testing.T) {
+ db := testutil.SetupTestDB(t, &entity.Task{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ svc := NewMemoryMessageService(NewMemoryService())
+ _, err := svc.extractMemoryTask(t.Context(), &entity.MemoryTask{MemoryID: "memory-1"}, MemoryMessage{})
+ if err == nil || errors.Is(err, errPermanentMemoryTask) {
+ t.Fatalf("extractMemoryTask error = %v, want retryable database error", err)
+ }
+}
+
+// TestExtractByLLMMarksMissingModelPermanent verifies a deleted model
+// reference terminates its durable task before any model call is attempted.
+func TestExtractByLLMMarksMissingModelPermanent(t *testing.T) {
+ db := testutil.SetupTestDB(t, &entity.TenantModel{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ svc := NewMemoryMessageService(NewMemoryService())
+ mem := &CreateMemoryResponse{Memory: entity.Memory{
+ TenantID: "tenant-1",
+ LLMID: "deleted-model-id",
+ }}
+ _, err := svc.extractByLLM(t.Context(), mem, []string{"semantic"}, MemoryMessage{}, "task-missing-model")
+ if !errors.Is(err, errPermanentMemoryTask) || !errors.Is(err, errModelConfigUnavailable) {
+ t.Fatalf("extractByLLM error = %v, want permanent missing-model error", err)
+ }
+}
+
+// TestExtractByLLMMarksInvalidModelConfigPermanent verifies malformed
+// provider-instance configuration terminates the durable task.
+func TestExtractByLLMMarksInvalidModelConfigPermanent(t *testing.T) {
+ db := testutil.SetupTestDB(t,
+ &entity.TenantModel{},
+ &entity.TenantModelProvider{},
+ &entity.TenantModelInstance{},
+ )
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ if err := db.Create(&entity.TenantModelProvider{
+ ID: "provider-1",
+ ProviderName: "OpenAI",
+ TenantID: "tenant-1",
+ }).Error; err != nil {
+ t.Fatalf("create model provider: %v", err)
+ }
+ if err := db.Create(&entity.TenantModelInstance{
+ ID: "instance-1",
+ InstanceName: "default",
+ ProviderID: "provider-1",
+ APIKey: "test-key",
+ Status: "active",
+ Extra: "{",
+ }).Error; err != nil {
+ t.Fatalf("create model instance: %v", err)
+ }
+
+ svc := NewMemoryMessageService(NewMemoryService())
+ mem := &CreateMemoryResponse{Memory: entity.Memory{
+ TenantID: "tenant-1",
+ LLMID: "gpt-4o@default@OpenAI",
+ }}
+ _, err := svc.extractByLLM(t.Context(), mem, []string{"semantic"}, MemoryMessage{}, "task-invalid-model-config")
+ if !errors.Is(err, errPermanentMemoryTask) || !errors.Is(err, errModelConfigUnavailable) {
+ t.Fatalf("extractByLLM error = %v, want permanent invalid-model-config error", err)
+ }
+}
+
+// TestHandleSaveToMemoryTaskResumesStoredCheckpoint verifies durable state,
+// rather than stale UI progress, determines where execution resumes.
+func TestHandleSaveToMemoryTaskResumesStoredCheckpoint(t *testing.T) {
+ pinMemoryNow(t, time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC))
+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ progressMsg := "stale UI state"
+ if err := db.Create(&entity.Task{
+ ID: "task-1",
+ DocID: "memory-1",
+ TaskType: "memory",
+ Progress: -1,
+ ProgressMsg: &progressMsg,
+ }).Error; err != nil {
+ t.Fatalf("create generic task: %v", err)
+ }
+ if err := db.Create(&entity.MemoryTask{
+ TaskID: "task-1",
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{},
+ State: entity.MemoryTaskStateStored,
+ }).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+
+ svc := NewMemoryMessageService(nil)
+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-1", "worker-1")
+ if err != nil || disposition != MemoryTaskAcknowledge {
+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v", disposition, err)
+ }
+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-1")
+ if err != nil {
+ t.Fatalf("load memory task: %v", err)
+ }
+ if stored.State != entity.MemoryTaskStateCompleted {
+ t.Fatalf("memory task state = %q, want completed", stored.State)
+ }
+ var task entity.Task
+ if err = db.First(&task, "id = ?", "task-1").Error; err != nil {
+ t.Fatalf("load generic task: %v", err)
+ }
+ if task.Progress != 1 {
+ t.Fatalf("generic task progress = %v, want 1", task.Progress)
+ }
+}
+
+// TestHandleSaveToMemoryTaskRetriesCompletionFromStored verifies a failed UI
+// progress write leaves the durable checkpoint at stored and the next attempt
+// performs only the final transaction.
+func TestHandleSaveToMemoryTaskRetriesCompletionFromStored(t *testing.T) {
+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)
+ originalNow := memoryNow
+ memoryNow = func() time.Time { return now }
+ t.Cleanup(func() { memoryNow = originalNow })
+
+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ progressMsg := "stored"
+ if err := db.Create(&entity.Task{
+ ID: "task-completion-retry",
+ DocID: "memory-1",
+ TaskType: "memory",
+ Progress: 0.5,
+ ProgressMsg: &progressMsg,
+ }).Error; err != nil {
+ t.Fatalf("create generic task: %v", err)
+ }
+ if err := db.Create(&entity.MemoryTask{
+ TaskID: "task-completion-retry",
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{},
+ State: entity.MemoryTaskStateStored,
+ }).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+ if err := db.Exec(`
+ CREATE TRIGGER fail_memory_task_completion_progress
+ BEFORE UPDATE ON task
+ BEGIN
+ SELECT RAISE(FAIL, 'forced completion progress failure');
+ END
+ `).Error; err != nil {
+ t.Fatalf("create update trigger: %v", err)
+ }
+
+ svc := NewMemoryMessageService(nil)
+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-completion-retry", "worker-1")
+ if err == nil || disposition != MemoryTaskAcknowledge {
+ t.Fatalf("first attempt disposition=%v err=%v, want acknowledged scheduled retry", disposition, err)
+ }
+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-completion-retry")
+ if err != nil {
+ t.Fatalf("load stored checkpoint: %v", err)
+ }
+ if stored.State != entity.MemoryTaskStateStored || stored.NextRetryAt == nil {
+ t.Fatalf("first attempt state/retry = %q/%v, want stored with retry", stored.State, stored.NextRetryAt)
+ }
+ if err = db.Exec("DROP TRIGGER fail_memory_task_completion_progress").Error; err != nil {
+ t.Fatalf("drop update trigger: %v", err)
+ }
+ now = *stored.NextRetryAt
+
+ disposition, err = svc.HandleSaveToMemoryTask(t.Context(), "task-completion-retry", "worker-2")
+ if err != nil || disposition != MemoryTaskAcknowledge {
+ t.Fatalf("retry disposition=%v err=%v, want completed acknowledgement", disposition, err)
+ }
+ completed, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-completion-retry")
+ if err != nil {
+ t.Fatalf("load completed task: %v", err)
+ }
+ if completed.State != entity.MemoryTaskStateCompleted {
+ t.Fatalf("retry state = %q, want completed", completed.State)
+ }
+}
+
+// TestHandleSaveToMemoryTaskResumesExtractedWithoutLLM verifies an extracted
+// checkpoint skips extraction even when no memory/LLM dependency is available.
+func TestHandleSaveToMemoryTaskResumesExtractedWithoutLLM(t *testing.T) {
+ pinMemoryNow(t, time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC))
+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ progressMsg := "extracted"
+ if err := db.Create(&entity.Task{
+ ID: "task-extracted-resume",
+ DocID: "memory-1",
+ TaskType: "memory",
+ Progress: 0.5,
+ ProgressMsg: &progressMsg,
+ }).Error; err != nil {
+ t.Fatalf("create generic task: %v", err)
+ }
+ if err := db.Create(&entity.MemoryTask{
+ TaskID: "task-extracted-resume",
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{"agent_id": "agent-1"},
+ State: entity.MemoryTaskStateExtracted,
+ Extraction: entity.JSONSlice{},
+ }).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+
+ svc := NewMemoryMessageService(nil)
+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-extracted-resume", "worker-1")
+ if err != nil || disposition != MemoryTaskAcknowledge {
+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v", disposition, err)
+ }
+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-extracted-resume")
+ if err != nil {
+ t.Fatalf("load memory task: %v", err)
+ }
+ if stored.State != entity.MemoryTaskStateCompleted {
+ t.Fatalf("memory task state = %q, want completed", stored.State)
+ }
+}
+
+// TestHandleSaveToMemoryTaskLeavesActiveLeaseUnsettled verifies an early
+// redelivery is preserved while another worker still owns the durable lease.
+func TestHandleSaveToMemoryTaskLeavesActiveLeaseUnsettled(t *testing.T) {
+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)
+ pinMemoryNow(t, now)
+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ if err := db.Create(&entity.MemoryTask{
+ TaskID: "task-live-lease",
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{},
+ State: entity.MemoryTaskStatePending,
+ }).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+ svc := NewMemoryMessageService(nil)
+ if _, acquired, err := svc.memoryTaskDAO.Claim(t.Context(), db, "task-live-lease", "worker-1", now, memoryTaskLeaseTTL); err != nil || !acquired {
+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)
+ }
+
+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-live-lease", "worker-2")
+ if err != nil || disposition != MemoryTaskLeaveUnsettled {
+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want unsettled live-lease delivery", disposition, err)
+ }
+}
+
+// TestHandleSaveToMemoryTaskFailsTaskMissingDurableState verifies a known
+// unfinished memory task is made visibly terminal instead of silently lost.
+func TestHandleSaveToMemoryTaskFailsTaskMissingDurableState(t *testing.T) {
+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ progressMsg := "queued"
+ if err := db.Create(&entity.Task{
+ ID: "task-missing-state",
+ DocID: "memory-1",
+ TaskType: common.TaskTypeMemory,
+ Progress: 0,
+ ProgressMsg: &progressMsg,
+ }).Error; err != nil {
+ t.Fatalf("create generic task: %v", err)
+ }
+
+ svc := NewMemoryMessageService(nil)
+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-missing-state", "worker-1")
+ if err == nil || disposition != MemoryTaskAcknowledge || !strings.Contains(err.Error(), "missing durable execution state") {
+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want acknowledged explicit failure", disposition, err)
+ }
+ var task entity.Task
+ if err = db.First(&task, "id = ?", "task-missing-state").Error; err != nil {
+ t.Fatalf("load generic task: %v", err)
+ }
+ if task.Progress != -1 || task.ProgressMsg == nil || !strings.Contains(*task.ProgressMsg, "save the message again") {
+ t.Fatalf("generic task progress/message = %v/%v, want visible upgrade failure", task.Progress, task.ProgressMsg)
+ }
+}
+
+// TestHandleSaveToMemoryTaskAcknowledgesUnknownID verifies a delivery without
+// either durable or generic task state remains distinguishable from known work
+// that must be projected as failed.
+func TestHandleSaveToMemoryTaskAcknowledgesUnknownID(t *testing.T) {
+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ svc := NewMemoryMessageService(nil)
+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "unknown-task", "worker-1")
+ if err == nil || disposition != MemoryTaskAcknowledge || !strings.Contains(err.Error(), "is not found") {
+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want acknowledged unknown task", disposition, err)
+ }
+}
+
+// TestHandleSaveToMemoryTaskDoesNotReportUnpersistedExtraction verifies a
+// failed extraction checkpoint leaves the best-effort UI projection behind it.
+func TestHandleSaveToMemoryTaskDoesNotReportUnpersistedExtraction(t *testing.T) {
+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)
+ pinMemoryNow(t, now)
+ db := testutil.SetupTestDB(t, &entity.User{}, &entity.Memory{}, &entity.Task{}, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ if err := db.Create(&entity.Memory{
+ ID: "memory-raw",
+ Name: "raw memory",
+ TenantID: "tenant-1",
+ MemoryType: dao.MemoryTypeRaw,
+ StorageType: "table",
+ Permissions: "me",
+ ForgettingPolicy: "FIFO",
+ }).Error; err != nil {
+ t.Fatalf("create memory: %v", err)
+ }
+ progressMsg := "queued"
+ if err := db.Create(&entity.Task{
+ ID: "task-checkpoint-failure",
+ DocID: "memory-raw",
+ TaskType: common.TaskTypeMemory,
+ ProgressMsg: &progressMsg,
+ }).Error; err != nil {
+ t.Fatalf("create generic task: %v", err)
+ }
+ if err := db.Create(&entity.MemoryTask{
+ TaskID: "task-checkpoint-failure",
+ MemoryID: "memory-raw",
+ SourceID: 42,
+ Input: entity.JSONMap{"agent_id": "agent-1"},
+ State: entity.MemoryTaskStatePending,
+ }).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+ if err := db.Exec(`
+ CREATE TRIGGER fail_memory_extraction_checkpoint
+ BEFORE UPDATE ON memory_task
+ WHEN NEW.state = 'extracted'
+ BEGIN
+ SELECT RAISE(FAIL, 'forced extraction checkpoint failure');
+ END
+ `).Error; err != nil {
+ t.Fatalf("create checkpoint trigger: %v", err)
+ }
+
+ svc := NewMemoryMessageService(NewMemoryService())
+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-checkpoint-failure", "worker-1")
+ if err == nil || disposition != MemoryTaskAcknowledge {
+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want acknowledged scheduled retry", disposition, err)
+ }
+ var task entity.Task
+ if err = db.First(&task, "id = ?", "task-checkpoint-failure").Error; err != nil {
+ t.Fatalf("load generic task: %v", err)
+ }
+ if task.Progress != 0 || task.ProgressMsg == nil || *task.ProgressMsg != "queued" {
+ t.Fatalf("generic task progress/message = %v/%v, want unchanged before checkpoint", task.Progress, task.ProgressMsg)
+ }
+}
+
+// TestPersistMemoryTaskFailureSchedulesRetry verifies a transient execution
+// failure is durably scheduled before the current delivery is acknowledged.
+func TestPersistMemoryTaskFailureSchedulesRetry(t *testing.T) {
+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)
+ pinMemoryNow(t, now)
+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ if err := db.Create(&entity.MemoryTask{
+ TaskID: "task-retry",
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{},
+ State: entity.MemoryTaskStatePending,
+ }).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+
+ svc := NewMemoryMessageService(nil)
+ claimed, acquired, err := svc.memoryTaskDAO.Claim(t.Context(), db, "task-retry", "worker-1", now, memoryTaskLeaseTTL)
+ if err != nil || !acquired {
+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)
+ }
+ disposition, err := svc.persistMemoryTaskFailure(t.Context(), claimed, "worker-1", errors.New("temporary failure"))
+ if err == nil || disposition != MemoryTaskAcknowledge {
+ t.Fatalf("persistMemoryTaskFailure disposition=%v err=%v, want acknowledged failure", disposition, err)
+ }
+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-retry")
+ if err != nil {
+ t.Fatalf("load memory task: %v", err)
+ }
+ if stored.State != entity.MemoryTaskStatePending {
+ t.Fatalf("memory task state = %q, want pending", stored.State)
+ }
+ wantRetryAt := now.Add(memoryTaskRetryInitialDelay)
+ if stored.NextRetryAt == nil || !stored.NextRetryAt.Equal(wantRetryAt) {
+ t.Fatalf("next retry at = %v, want %v", stored.NextRetryAt, wantRetryAt)
+ }
+ if stored.LeaseOwner != "" || stored.LeaseExpiresAt != nil {
+ t.Fatalf("lease = %q/%v, want released", stored.LeaseOwner, stored.LeaseExpiresAt)
+ }
+ if stored.AttemptCount != 1 || stored.LastError == "" {
+ t.Fatalf("attempt/error = %d/%q, want one failed attempt", stored.AttemptCount, stored.LastError)
+ }
+}
+
+// TestPersistMemoryTaskFailureStopsAtMaxAttempts verifies a retryable error
+// becomes terminal once the claimed-attempt limit is reached.
+func TestPersistMemoryTaskFailureStopsAtMaxAttempts(t *testing.T) {
+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)
+ pinMemoryNow(t, now)
+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ progressMsg := "queued"
+ if err := db.Create(&entity.Task{
+ ID: "task-max-attempts",
+ DocID: "memory-1",
+ TaskType: common.TaskTypeMemory,
+ ProgressMsg: &progressMsg,
+ }).Error; err != nil {
+ t.Fatalf("create generic task: %v", err)
+ }
+ if err := db.Create(&entity.MemoryTask{
+ TaskID: "task-max-attempts",
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{},
+ State: entity.MemoryTaskStatePending,
+ AttemptCount: memoryTaskMaxAttempts - 1,
+ }).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+
+ svc := NewMemoryMessageService(nil)
+ claimed, acquired, err := svc.memoryTaskDAO.Claim(t.Context(), db, "task-max-attempts", "worker-1", now, memoryTaskLeaseTTL)
+ if err != nil || !acquired {
+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)
+ }
+ disposition, err := svc.persistMemoryTaskFailure(t.Context(), claimed, "worker-1", errors.New("empty response from chat model"))
+ if err == nil || disposition != MemoryTaskAcknowledge || !strings.Contains(err.Error(), "failed after 10 attempts") {
+ t.Fatalf("persistMemoryTaskFailure disposition=%v err=%v, want acknowledged terminal failure", disposition, err)
+ }
+
+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, claimed.TaskID)
+ if err != nil {
+ t.Fatalf("load memory task: %v", err)
+ }
+ if stored.State != entity.MemoryTaskStateFailed || stored.AttemptCount != memoryTaskMaxAttempts || stored.NextRetryAt != nil {
+ t.Fatalf("memory task state/attempt/retry = %q/%d/%v, want failed/%d/nil", stored.State, stored.AttemptCount, stored.NextRetryAt, memoryTaskMaxAttempts)
+ }
+ var task entity.Task
+ if err = db.First(&task, "id = ?", claimed.TaskID).Error; err != nil {
+ t.Fatalf("load generic task: %v", err)
+ }
+ if task.Progress != -1 || task.ProgressMsg == nil || !strings.Contains(*task.ProgressMsg, "failed after 10 attempts") {
+ t.Fatalf("generic task progress/message = %v/%v, want terminal retry-limit error", task.Progress, task.ProgressMsg)
+ }
+}
+
+// TestPersistMemoryTaskFailureLeavesDeliveryUnsettledWhenRetryWriteFails
+// verifies the broker message is preserved when no durable retry decision can
+// be recorded.
+func TestPersistMemoryTaskFailureLeavesDeliveryUnsettledWhenRetryWriteFails(t *testing.T) {
+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)
+ pinMemoryNow(t, now)
+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ if err := db.Create(&entity.MemoryTask{
+ TaskID: "task-retry-write-failure",
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{},
+ State: entity.MemoryTaskStatePending,
+ }).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+
+ svc := NewMemoryMessageService(nil)
+ claimed, acquired, err := svc.memoryTaskDAO.Claim(t.Context(), db, "task-retry-write-failure", "worker-1", now, memoryTaskLeaseTTL)
+ if err != nil || !acquired {
+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)
+ }
+ if err = db.Exec(`
+ CREATE TRIGGER fail_memory_task_retry_update
+ BEFORE UPDATE ON memory_task
+ BEGIN
+ SELECT RAISE(FAIL, 'forced retry update failure');
+ END
+ `).Error; err != nil {
+ t.Fatalf("create retry trigger: %v", err)
+ }
+
+ disposition, err := svc.persistMemoryTaskFailure(t.Context(), claimed, "worker-1", errors.New("temporary failure"))
+ if err == nil || disposition != MemoryTaskLeaveUnsettled {
+ t.Fatalf("persistMemoryTaskFailure disposition=%v err=%v, want unsettled failure", disposition, err)
+ }
+}
+
+// TestRunClaimedMemoryTaskPersistsPanic verifies a state-machine panic is
+// converted into the normal durable retry path after lease renewal stops.
+func TestRunClaimedMemoryTaskPersistsPanic(t *testing.T) {
+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)
+ pinMemoryNow(t, now)
+ pinMemoryTaskLeaseTimings(t, 10*time.Millisecond, 10*time.Millisecond)
+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ expiresAt := now.Add(time.Minute)
+ task := &entity.MemoryTask{
+ TaskID: "task-panic-cleanup",
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{},
+ State: entity.MemoryTaskStatePending,
+ LeaseOwner: "worker-1",
+ LeaseExpiresAt: &expiresAt,
+ }
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+
+ svc := NewMemoryMessageService(nil)
+ svc.resumeTask = func(context.Context, *entity.MemoryTask, string) error {
+ panic("simulated state-machine panic")
+ }
+ disposition, runErr := svc.runClaimedMemoryTask(t.Context(), task, "worker-1")
+ if runErr == nil || disposition != MemoryTaskAcknowledge || !strings.Contains(runErr.Error(), "panicked: simulated state-machine panic") {
+ t.Fatalf("runClaimedMemoryTask disposition=%v err=%v, want acknowledged persisted panic", disposition, runErr)
+ }
+
+ time.Sleep(3 * memoryTaskLeaseRenewInterval)
+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, task.TaskID)
+ if err != nil {
+ t.Fatalf("load memory task: %v", err)
+ }
+ if stored.State != entity.MemoryTaskStatePending || stored.NextRetryAt == nil || stored.LeaseOwner != "" || stored.LeaseExpiresAt != nil {
+ t.Fatalf("memory task after panic = %+v, want pending scheduled retry with released lease", stored)
+ }
+ if !strings.Contains(stored.LastError, "panicked: simulated state-machine panic") {
+ t.Fatalf("last error = %q, want persisted panic", stored.LastError)
+ }
+}
+
+// TestRenewMemoryTaskLeaseRetriesTransientError verifies one failed renewal
+// does not discard in-flight work while the last confirmed lease is healthy.
+func TestRenewMemoryTaskLeaseRetriesTransientError(t *testing.T) {
+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)
+ pinMemoryNow(t, now)
+ pinMemoryTaskLeaseTimings(t, 5*time.Millisecond, 10*time.Millisecond)
+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ expiresAt := now.Add(time.Minute)
+ if err := db.Create(&entity.MemoryTask{
+ TaskID: "task-renew-transient",
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{},
+ State: entity.MemoryTaskStatePending,
+ LeaseOwner: "worker-1",
+ LeaseExpiresAt: &expiresAt,
+ }).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+ var attempts atomic.Int32
+ if err := db.Callback().Update().Before("gorm:update").Register("fail_first_memory_task_lease_renewal", func(tx *gorm.DB) {
+ if attempts.Add(1) == 1 {
+ tx.AddError(errors.New("transient renewal failure"))
+ }
+ }); err != nil {
+ t.Fatalf("register renewal callback: %v", err)
+ }
+
+ svc := NewMemoryMessageService(nil)
+ runCtx, cancel := context.WithCancel(t.Context())
+ renewErr := make(chan error, 1)
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ svc.renewMemoryTaskLease(runCtx, cancel, "task-renew-transient", "worker-1", expiresAt, renewErr)
+ }()
+
+ wantExpiresAt := now.Add(memoryTaskLeaseTTL)
+ deadline := time.NewTimer(time.Second)
+ defer deadline.Stop()
+ poll := time.NewTicker(time.Millisecond)
+ defer poll.Stop()
+ for {
+ select {
+ case <-deadline.C:
+ cancel()
+ <-done
+ t.Fatal("lease was not renewed after a transient failure")
+ case <-poll.C:
+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-renew-transient")
+ if err != nil {
+ t.Fatalf("load memory task: %v", err)
+ }
+ if stored.LeaseExpiresAt == nil || !stored.LeaseExpiresAt.Equal(wantExpiresAt) {
+ continue
+ }
+ if attempts.Load() < 2 {
+ t.Fatalf("renewal attempts = %d, want at least 2", attempts.Load())
+ }
+ if runCtx.Err() != nil {
+ t.Fatalf("run context canceled after recoverable renewal error: %v", runCtx.Err())
+ }
+ select {
+ case err := <-renewErr:
+ t.Fatalf("renewal reported terminal error after recovery: %v", err)
+ default:
+ }
+ cancel()
+ <-done
+ return
+ }
+ }
+}
+
+// TestRenewMemoryTaskLeaseTimesOutNearExpiry verifies an unresponsive renewal
+// cancels execution when no full renewal interval remains on the lease.
+func TestRenewMemoryTaskLeaseTimesOutNearExpiry(t *testing.T) {
+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)
+ pinMemoryNow(t, now)
+ pinMemoryTaskLeaseTimings(t, 5*time.Millisecond, 10*time.Millisecond)
+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ expiresAt := now.Add(memoryTaskLeaseRenewInterval)
+ if err := db.Create(&entity.MemoryTask{
+ TaskID: "task-renew-timeout",
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{},
+ State: entity.MemoryTaskStatePending,
+ LeaseOwner: "worker-1",
+ LeaseExpiresAt: &expiresAt,
+ }).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+ if err := db.Callback().Update().Before("gorm:update").Register("block_memory_task_lease_renewal", func(tx *gorm.DB) {
+ <-tx.Statement.Context.Done()
+ tx.AddError(tx.Statement.Context.Err())
+ }); err != nil {
+ t.Fatalf("register blocking update callback: %v", err)
+ }
+
+ svc := NewMemoryMessageService(nil)
+ runCtx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ renewErr := make(chan error, 1)
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ svc.renewMemoryTaskLease(runCtx, cancel, "task-renew-timeout", "worker-1", expiresAt, renewErr)
+ }()
+
+ select {
+ case <-done:
+ case <-time.After(time.Second):
+ t.Fatal("renewMemoryTaskLease did not stop after update timeout")
+ }
+ if runCtx.Err() == nil {
+ t.Fatal("run context was not canceled after lease renewal timeout")
+ }
+ select {
+ case err := <-renewErr:
+ if err == nil || !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) {
+ t.Fatalf("renew error = %v, want deadline exceeded", err)
+ }
+ default:
+ t.Fatal("renewMemoryTaskLease did not report the timeout")
+ }
+}
+
+// TestHandleSaveToMemoryTaskMarksInvalidInputFailed verifies corrupt durable
+// input reaches the terminal state and updates the UI progress projection.
+func TestHandleSaveToMemoryTaskMarksInvalidInputFailed(t *testing.T) {
+ pinMemoryNow(t, time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC))
+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ progressMsg := "queued"
+ if err := db.Create(&entity.Task{
+ ID: "task-invalid",
+ DocID: "memory-1",
+ TaskType: "memory",
+ Progress: 0,
+ ProgressMsg: &progressMsg,
+ }).Error; err != nil {
+ t.Fatalf("create generic task: %v", err)
+ }
+ if err := db.Create(&entity.MemoryTask{
+ TaskID: "task-invalid",
+ MemoryID: "memory-1",
+ SourceID: 42,
+ Input: entity.JSONMap{},
+ State: entity.MemoryTaskStatePending,
+ }).Error; err != nil {
+ t.Fatalf("create memory task: %v", err)
+ }
+
+ svc := NewMemoryMessageService(nil)
+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-invalid", "worker-1")
+ if err == nil || disposition != MemoryTaskAcknowledge {
+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want acknowledged terminal failure", disposition, err)
+ }
+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-invalid")
+ if err != nil {
+ t.Fatalf("load memory task: %v", err)
+ }
+ if stored.State != entity.MemoryTaskStateFailed || stored.NextRetryAt != nil {
+ t.Fatalf("memory task state/retry = %q/%v, want failed without retry", stored.State, stored.NextRetryAt)
+ }
+ var task entity.Task
+ if err = db.First(&task, "id = ?", "task-invalid").Error; err != nil {
+ t.Fatalf("load generic task: %v", err)
+ }
+ if task.Progress != -1 || task.ProgressMsg == nil || !strings.Contains(*task.ProgressMsg, "agent_id is required") {
+ t.Fatalf("generic task progress/message = %v/%v, want terminal input error", task.Progress, task.ProgressMsg)
+ }
+}
+
+// TestMemoryTaskRetryDelay verifies retries use bounded exponential backoff.
+func TestMemoryTaskRetryDelay(t *testing.T) {
+ for _, test := range []struct {
+ attemptCount int
+ want time.Duration
+ }{
+ {attemptCount: 0, want: 5 * time.Second},
+ {attemptCount: 1, want: 5 * time.Second},
+ {attemptCount: 2, want: 10 * time.Second},
+ {attemptCount: 7, want: 5 * time.Minute},
+ {attemptCount: 100, want: 5 * time.Minute},
+ } {
+ if got := memoryTaskRetryDelay(test.attemptCount); got != test.want {
+ t.Errorf("memoryTaskRetryDelay(%d) = %v, want %v", test.attemptCount, got, test.want)
+ }
+ }
+}
+
+// TestUpdateTaskProgressReturnsPersistenceError verifies UI projection writes
+// surface persistence failures without becoming execution checkpoints.
func TestUpdateTaskProgressReturnsPersistenceError(t *testing.T) {
db := testutil.SetupTestDB(t, &entity.IngestionTask{})
cleanup := testutil.ReplaceDBForTest(t, db)
diff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.go
index f77b688a1e..b42a892c76 100644
--- a/internal/service/memory_message_service.go
+++ b/internal/service/memory_message_service.go
@@ -47,25 +47,26 @@
// 1. For each memory id: look up the Memory (via MemoryService).
// 2. Generate a raw_message_id from Redis auto-increment (namespace "memory").
// 3. Build the raw_message envelope (mirrors Python:344-386).
-// 4. Call embed_and_save on the memory + [raw_message].
-// 5. Insert a Task row in the task table for the async extractor.
-// 6. Return not-found + failed lists.
+// 4. Insert the UI Task and durable MemoryTask rows atomically.
+// 5. Call embed_and_save on the memory + [raw_message].
+// 6. Publish a task-id wake-up for the async extractor.
+// 7. Return not-found + failed lists.
package service
import (
"context"
- "encoding/json"
"errors"
"fmt"
"time"
"ragflow/internal/common"
"ragflow/internal/dao"
- "ragflow/internal/engine"
redisengine "ragflow/internal/engine/redis"
"ragflow/internal/entity"
models "ragflow/internal/entity/models"
"ragflow/internal/utility"
+
+ "gorm.io/gorm"
)
// MemoryMessage is the wire shape for QueueSaveToMemoryTask. It
@@ -103,8 +104,11 @@ type QueueSaveResult struct {
// MemoryMessageService is the Go port of
// api.db.joint_services.memory_message_service.
type MemoryMessageService struct {
- memories *MemoryService
- taskDAO *dao.TaskDAO
+ memories *MemoryService
+ taskDAO *dao.TaskDAO
+ memoryTaskDAO *dao.MemoryTaskDAO
+ taskPublisher TaskPublisher
+ resumeTask func(context.Context, *entity.MemoryTask, string) error
}
// NewMemoryMessageService constructs a service bound to the
@@ -113,8 +117,10 @@ type MemoryMessageService struct {
// `component.SetMemorySaver(...)` at boot.
func NewMemoryMessageService(memories *MemoryService) *MemoryMessageService {
return &MemoryMessageService{
- memories: memories,
- taskDAO: dao.NewTaskDAO(),
+ memories: memories,
+ taskDAO: dao.NewTaskDAO(),
+ memoryTaskDAO: dao.NewMemoryTaskDAO(),
+ taskPublisher: NewMessageQueueTaskPublisher(),
}
}
@@ -144,7 +150,14 @@ func (s *MemoryMessageService) QueueSaveToMemoryTask(ctx context.Context, memory
// (1) Look up the memory (no access control — trusted internal queue processing).
mem, err := s.memories.getMemoryConfig(ctx, memoryID)
if err != nil {
- res.NotFound = append(res.NotFound, memoryID)
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ res.NotFound = append(res.NotFound, memoryID)
+ } else {
+ res.Failed = append(res.Failed, MemoryFailure{
+ MemoryID: memoryID,
+ FailMsg: err.Error(),
+ })
+ }
continue
}
// (2) + (3) build the raw_message envelope. The Go port
@@ -153,33 +166,67 @@ func (s *MemoryMessageService) QueueSaveToMemoryTask(ctx context.Context, memory
// schema changes.
rawMessageID := generateRawMessageID(ctx)
rawMessage := buildRawMessage(rawMessageID, memoryID, msg)
-
- if err := s.embedAndSave(ctx, mem, rawMessage); err != nil {
+ task, memoryTask := buildMemoryTaskRecords(rawMessageID, memoryID, msg)
+ if err := s.insertMemoryTask(ctx, task, memoryTask); err != nil {
res.Failed = append(res.Failed, MemoryFailure{
MemoryID: memoryID,
- FailMsg: err.Error(),
+ FailMsg: fmt.Sprintf("task insert: %s", err.Error()),
})
continue
}
- task := buildTaskRow(rawMessageID, memoryID)
- if err := s.insertTask(ctx, task); err != nil {
+ if err := s.embedAndSave(ctx, mem, rawMessage); err != nil {
+ failure := fmt.Errorf("store raw memory message: %w", err)
+ failed, failErr := s.memoryTaskDAO.MarkUnclaimedFailed(ctx, dao.DB, task.ID, failure.Error())
+ if failErr != nil {
+ failure = errors.Join(failure, fmt.Errorf("mark memory task failed: %w", failErr))
+ } else if !failed {
+ failure = errors.Join(failure, errors.New("memory task was claimed before raw message storage completed"))
+ }
res.Failed = append(res.Failed, MemoryFailure{
MemoryID: memoryID,
- FailMsg: fmt.Sprintf("task insert: %s", err.Error()),
+ FailMsg: failure.Error(),
})
continue
}
- if err = queueMemoryTask(ctx, fmt.Sprint(task["id"]), memoryID, mem.TenantID, rawMessageID, msg); err != nil {
- res.Failed = append(res.Failed, MemoryFailure{
- MemoryID: memoryID,
- FailMsg: err.Error(),
- })
+
+ if err = publishMemoryTaskWakeup(s.taskPublisher, task.ID); err != nil {
+ common.Warn(fmt.Sprintf("memory: initial task wake-up failed; reconciler will retry: %v", err))
}
}
return res, nil
}
+// ReconcileMemoryTasks publishes wake-ups for due, unleased durable memory
+// tasks. Publishing is idempotent because workers must claim the DB lease
+// before executing any stage.
+func (s *MemoryMessageService) ReconcileMemoryTasks(ctx context.Context, limit int) error {
+ if s == nil {
+ return errors.New("memory: nil MemoryMessageService")
+ }
+ if s.memoryTaskDAO == nil {
+ s.memoryTaskDAO = dao.NewMemoryTaskDAO()
+ }
+ if s.taskPublisher == nil {
+ return errors.New("memory task publisher is not initialized")
+ }
+
+ tasks, err := s.memoryTaskDAO.ListDue(ctx, dao.DB, memoryNow(), limit)
+ if err != nil {
+ return fmt.Errorf("memory: list due tasks: %w", err)
+ }
+ var reconcileErr error
+ for _, task := range tasks {
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return errors.Join(reconcileErr, ctxErr)
+ }
+ if err = publishMemoryTaskWakeup(s.taskPublisher, task.TaskID); err != nil {
+ reconcileErr = errors.Join(reconcileErr, err)
+ }
+ }
+ return reconcileErr
+}
+
// generateRawMessageID returns the Redis auto-increment id used by the Python
// side (`REDIS_CONN.generate_auto_increment_id(namespace="memory")`).
func generateRawMessageID(ctx context.Context) int64 {
@@ -222,17 +269,37 @@ func buildRawMessage(
}
}
-// buildTaskRow constructs the Task row the async extractor polls.
-func buildTaskRow(rawMessageID int64, memoryID string) map[string]any {
- return map[string]any{
- "id": newUUIDString(),
- "doc_id": memoryID,
- "task_type": "memory",
- "progress": 0.0,
- "progress_msg": "",
- "begin_at": time.Now(),
- "digest": fmt.Sprintf("%d", rawMessageID),
+// buildMemoryTaskRecords constructs the UI projection and authoritative
+// execution record that are inserted atomically before publishing a wake-up.
+func buildMemoryTaskRecords(rawMessageID int64, memoryID string, msg MemoryMessage) (*entity.Task, *entity.MemoryTask) {
+ taskID := newUUIDString()
+ progressMsg := ""
+ digest := fmt.Sprintf("%d", rawMessageID)
+ beginAt := time.Now()
+ task := &entity.Task{
+ ID: taskID,
+ DocID: memoryID,
+ TaskType: common.TaskTypeMemory,
+ Progress: 0,
+ ProgressMsg: &progressMsg,
+ BeginAt: &beginAt,
+ Digest: &digest,
}
+ memoryTask := &entity.MemoryTask{
+ TaskID: taskID,
+ MemoryID: memoryID,
+ SourceID: rawMessageID,
+ Input: entity.JSONMap{
+ "user_id": msg.UserID,
+ "agent_id": msg.AgentID,
+ "session_id": msg.SessionID,
+ "user_input": msg.UserInput,
+ "agent_response": msg.AgentResponse,
+ },
+ State: entity.MemoryTaskStatePending,
+ LastError: "",
+ }
+ return task, memoryTask
}
func (s *MemoryMessageService) embedAndSave(ctx context.Context, mem *CreateMemoryResponse, rawMessage map[string]any) error {
@@ -305,14 +372,15 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Cr
return nil
}
-func (s *MemoryMessageService) insertTask(ctx context.Context, row map[string]any) error {
+// insertMemoryTask persists both task records atomically.
+func (s *MemoryMessageService) insertMemoryTask(ctx context.Context, task *entity.Task, memoryTask *entity.MemoryTask) error {
if s == nil {
return errors.New("nil MemoryMessageService")
}
- if s.taskDAO == nil {
- s.taskDAO = dao.NewTaskDAO()
+ if s.memoryTaskDAO == nil {
+ s.memoryTaskDAO = dao.NewMemoryTaskDAO()
}
- return s.taskDAO.Create(ctx, dao.DB, taskFromRow(row))
+ return s.memoryTaskDAO.CreateWithTask(ctx, dao.DB, task, memoryTask)
}
// newUUIDString is a thin wrapper so we can swap in a real UUID
@@ -322,67 +390,17 @@ func newUUIDString() string {
return utility.GenerateUUID()
}
-func taskFromRow(row map[string]any) *entity.Task {
- digest := fmt.Sprint(row["digest"])
- progressMsg := fmt.Sprint(row["progress_msg"])
- beginAt, _ := row["begin_at"].(time.Time)
- if beginAt.IsZero() {
- now := time.Now()
- beginAt = now
- }
- return &entity.Task{
- ID: fmt.Sprint(row["id"]),
- DocID: fmt.Sprint(row["doc_id"]),
- TaskType: fmt.Sprint(row["task_type"]),
- Progress: 0,
- ProgressMsg: &progressMsg,
- BeginAt: &beginAt,
- Digest: &digest,
- }
-}
-
-// queueMemoryTask publishes a memory-extraction task to NATS (tasks.RAGFLOW).
-// taskID is the durable task row's id (buildTaskRow) and the sole identity on
-// the envelope; the payload carries only business parameters — memory_id,
-// source_id, and the dialogue to extract. Identity is deliberately NOT
-// duplicated into the payload so the consumer never has two ids that could
-// disagree (the envelope TaskID is authoritative end to end).
-func queueMemoryTask(ctx context.Context, taskID, memoryID, tenantID string, rawMessageID int64, msg MemoryMessage) error {
- message := map[string]any{
- "memory_id": memoryID,
- "tenant_id": tenantID,
- "source_id": rawMessageID,
- "message_dict": map[string]any{
- "user_id": msg.UserID,
- "agent_id": msg.AgentID,
- "session_id": msg.SessionID,
- "user_input": msg.UserInput,
- "agent_response": msg.AgentResponse,
- },
- }
- // Publish the memory-extraction task to NATS (tasks.RAGFLOW) so it is
- // consumed by the Ingestor's shared consumer + worker pool, dispatched by
- // TaskType=="memory" in handleAndExecute. This keeps Go out of the Python
- // te.*.common Redis stream entirely, removing the cross-consumer
- // contention that previously stole Python dataflow tasks.
- mq := engine.GetMessageQueueEngine()
- if mq == nil {
- return errors.New("can't access message queue engine")
- }
- payload, err := json.Marshal(message)
- if err != nil {
- return fmt.Errorf("marshal memory task payload: %w", err)
+// publishMemoryTaskWakeup publishes only the durable task identity. Workers
+// load all execution input and checkpoints from memory_task.
+func publishMemoryTaskWakeup(publisher TaskPublisher, taskID string) error {
+ if publisher == nil {
+ return errors.New("memory task publisher is not initialized")
}
taskMessage := common.TaskMessage{
TaskID: taskID,
TaskType: common.TaskTypeMemory,
- Payload: payload,
- }
- tmPayload, err := json.Marshal(taskMessage)
- if err != nil {
- return fmt.Errorf("marshal memory task message: %w", err)
}
- if err = mq.PublishTask(common.TaskSubject, tmPayload); err != nil {
+ if err := publisher.PublishTaskMessage(common.TaskSubject, taskMessage); err != nil {
return fmt.Errorf("publish memory task %s: %w", taskID, err)
}
return nil
diff --git a/internal/service/memory_message_service_test.go b/internal/service/memory_message_service_test.go
index 66bad6d966..97d54993fa 100644
--- a/internal/service/memory_message_service_test.go
+++ b/internal/service/memory_message_service_test.go
@@ -25,6 +25,9 @@ import (
"strings"
"testing"
"time"
+
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/testutil"
)
// TestQueueSaveToMemoryTask_NilService: a nil receiver surfaces
@@ -67,6 +70,94 @@ func TestQueueSaveToMemoryTask_MissingAgentID(t *testing.T) {
}
}
+// TestQueueSaveToMemoryTaskClassifiesMemoryLookupErrors keeps confirmed missing
+// memories separate from transient database failures.
+func TestQueueSaveToMemoryTaskClassifiesMemoryLookupErrors(t *testing.T) {
+ t.Run("missing memory", func(t *testing.T) {
+ db := testutil.SetupTestDB(t, &entity.Memory{}, &entity.User{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ res, err := NewMemoryMessageService(NewMemoryService()).QueueSaveToMemoryTask(
+ t.Context(), []string{"missing-memory"}, MemoryMessage{AgentID: "agent-1"},
+ )
+ if err != nil {
+ t.Fatalf("QueueSaveToMemoryTask: %v", err)
+ }
+ if len(res.NotFound) != 1 || res.NotFound[0] != "missing-memory" || len(res.Failed) != 0 {
+ t.Fatalf("result = %+v, want missing memory in NotFound only", res)
+ }
+ })
+
+ t.Run("database failure", func(t *testing.T) {
+ db := testutil.SetupTestDB(t, &entity.Task{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ res, err := NewMemoryMessageService(NewMemoryService()).QueueSaveToMemoryTask(
+ t.Context(), []string{"memory-1"}, MemoryMessage{AgentID: "agent-1"},
+ )
+ if err != nil {
+ t.Fatalf("QueueSaveToMemoryTask: %v", err)
+ }
+ if len(res.NotFound) != 0 || len(res.Failed) != 1 || res.Failed[0].MemoryID != "memory-1" || res.Failed[0].FailMsg == "" {
+ t.Fatalf("result = %+v, want database lookup failure in Failed only", res)
+ }
+ })
+}
+
+// TestQueueSaveToMemoryTaskMarksDurableTaskFailedWhenRawStorageFails verifies
+// task state exists before raw storage and records a visible terminal outcome
+// when that storage cannot start.
+func TestQueueSaveToMemoryTaskMarksDurableTaskFailedWhenRawStorageFails(t *testing.T) {
+ db := testutil.SetupTestDB(t, &entity.User{}, &entity.Memory{}, &entity.Task{}, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ if err := db.Create(&entity.Memory{
+ ID: "memory-1",
+ Name: "memory",
+ TenantID: "tenant-1",
+ MemoryType: 1,
+ StorageType: "table",
+ Permissions: "me",
+ ForgettingPolicy: "FIFO",
+ }).Error; err != nil {
+ t.Fatalf("create memory: %v", err)
+ }
+ memorySvc := NewMemoryService()
+ memorySvc.docEngine = nil
+ svc := NewMemoryMessageService(memorySvc)
+ publisher := &recordingTaskPublisher{}
+ svc.taskPublisher = publisher
+
+ res, err := svc.QueueSaveToMemoryTask(t.Context(), []string{"memory-1"}, MemoryMessage{AgentID: "agent-1"})
+ if err != nil {
+ t.Fatalf("QueueSaveToMemoryTask: %v", err)
+ }
+ if len(res.Failed) != 1 || !strings.Contains(res.Failed[0].FailMsg, "message store is not initialized") {
+ t.Fatalf("result = %+v, want raw-storage failure", res)
+ }
+ if len(publisher.messages) != 0 {
+ t.Fatalf("published messages = %d, want none after raw-storage failure", len(publisher.messages))
+ }
+
+ var memoryTask entity.MemoryTask
+ if err = db.First(&memoryTask).Error; err != nil {
+ t.Fatalf("load memory task: %v", err)
+ }
+ if memoryTask.State != entity.MemoryTaskStateFailed || !strings.Contains(memoryTask.LastError, "message store is not initialized") {
+ t.Fatalf("memory task state/error = %q/%q, want failed raw-storage error", memoryTask.State, memoryTask.LastError)
+ }
+ var task entity.Task
+ if err = db.First(&task, "id = ?", memoryTask.TaskID).Error; err != nil {
+ t.Fatalf("load generic task: %v", err)
+ }
+ if task.Progress != -1 || task.ProgressMsg == nil || !strings.Contains(*task.ProgressMsg, "message store is not initialized") {
+ t.Fatalf("generic task progress/message = %v/%v, want visible raw-storage failure", task.Progress, task.ProgressMsg)
+ }
+}
+
// TestBuildRawMessage_ValidAtServerLocal: valid_at is stamped as a
// server-local wall-clock string, not UTC — otherwise memories asked at
// 10:05 local show up as 02:05. The clock is pinned to a fixed instant in a
@@ -137,37 +228,34 @@ func TestBuildRawMessage_EnvelopeShape(t *testing.T) {
}
}
-// TestBuildTaskRow_Shape: the task row mirrors the Python Task
-// entity's memory-task shape.
-func TestBuildTaskRow_Shape(t *testing.T) {
- row := buildTaskRow(99, "mem-1")
- if row["task_type"] != "memory" {
- t.Errorf("task_type = %v, want \"memory\"", row["task_type"])
+// TestBuildMemoryTaskRecordsShape verifies the UI task and durable execution
+// record carry the same identity and the worker input is persisted in the DB.
+func TestBuildMemoryTaskRecordsShape(t *testing.T) {
+ msg := MemoryMessage{UserID: "u1", AgentID: "a1", SessionID: "s1", UserInput: "hi", AgentResponse: "hello"}
+ task, memoryTask := buildMemoryTaskRecords(99, "mem-1", msg)
+ if task.TaskType != "memory" {
+ t.Errorf("task_type = %v, want \"memory\"", task.TaskType)
}
- if row["doc_id"] != "mem-1" {
- t.Errorf("doc_id = %v, want \"mem-1\"", row["doc_id"])
+ if task.DocID != "mem-1" {
+ t.Errorf("doc_id = %v, want \"mem-1\"", task.DocID)
}
- if row["progress"] != 0.0 {
- t.Errorf("progress = %v, want 0.0", row["progress"])
+ if task.Progress != 0.0 {
+ t.Errorf("progress = %v, want 0.0", task.Progress)
}
- if row["progress_msg"] != "" {
- t.Errorf("progress_msg = %v, want empty string", row["progress_msg"])
+ if task.ProgressMsg == nil || *task.ProgressMsg != "" {
+ t.Errorf("progress_msg = %v, want empty string", task.ProgressMsg)
}
- if row["digest"] != "99" {
- t.Errorf("digest = %v, want \"99\"", row["digest"])
+ if task.Digest == nil || *task.Digest != "99" {
+ t.Errorf("digest = %v, want \"99\"", task.Digest)
}
- if id, _ := row["id"].(string); len(id) != 32 {
- t.Errorf("id = %q, want 32-char uuid", id)
+ if len(task.ID) != 32 {
+ t.Errorf("id = %q, want 32-char uuid", task.ID)
}
-}
-
-func TestTaskFromRow_InitializesProgressMessage(t *testing.T) {
- task := taskFromRow(buildTaskRow(99, "mem-1"))
- if task.ProgressMsg == nil {
- t.Fatal("ProgressMsg is nil")
+ if memoryTask.TaskID != task.ID || memoryTask.MemoryID != "mem-1" || memoryTask.SourceID != 99 {
+ t.Fatalf("memory task identity = %+v, want task %s/mem-1/99", memoryTask, task.ID)
}
- if *task.ProgressMsg != "" {
- t.Fatalf("ProgressMsg = %q, want empty string", *task.ProgressMsg)
+ if memoryTask.State != "pending" || memoryTask.Input["agent_response"] != "hello" {
+ t.Fatalf("memory task execution data = %+v", memoryTask)
}
}
diff --git a/internal/service/memory_message_test.go b/internal/service/memory_message_test.go
index c268e2fcda..e8b97fa861 100644
--- a/internal/service/memory_message_test.go
+++ b/internal/service/memory_message_test.go
@@ -175,6 +175,8 @@ func setupMemoryMessageTestDB(t *testing.T) {
&entity.TenantModelProvider{},
&entity.TenantModelInstance{},
&entity.TenantModel{},
+ &entity.Task{},
+ &entity.MemoryTask{},
); err != nil {
t.Fatalf("failed to migrate memory test tables: %v", err)
}
diff --git a/internal/service/memory_queue_task_test.go b/internal/service/memory_queue_task_test.go
index 37f1e86d64..313b5997e3 100644
--- a/internal/service/memory_queue_task_test.go
+++ b/internal/service/memory_queue_task_test.go
@@ -4,11 +4,28 @@ import (
"strings"
"testing"
+ "ragflow/internal/common"
"ragflow/internal/engine"
natsengine "ragflow/internal/engine/nats"
)
-// queueMemoryTask is the agent-canvas memory-save publish path. When the
+// TestPublishMemoryTaskWakeupContainsOnlyDurableIdentity verifies NATS carries
+// no execution input that can diverge from the database record.
+func TestPublishMemoryTaskWakeupContainsOnlyDurableIdentity(t *testing.T) {
+ publisher := &recordingTaskPublisher{}
+ if err := publishMemoryTaskWakeup(publisher, "task-1"); err != nil {
+ t.Fatalf("publishMemoryTaskWakeup: %v", err)
+ }
+ if publisher.subject != common.TaskSubject || len(publisher.messages) != 1 {
+ t.Fatalf("published subject/messages = %q/%d", publisher.subject, len(publisher.messages))
+ }
+ message := publisher.messages[0]
+ if message.TaskID != "task-1" || message.TaskType != common.TaskTypeMemory {
+ t.Fatalf("published memory wake-up = %+v", message)
+ }
+}
+
+// publishMemoryTaskWakeup is the agent-canvas memory-save publish path. When the
// message queue engine is a NatsEngine whose Init failed at boot, publishing
// must surface a clean error (caught by the Message component's best-effort
// memory save) rather than panicking and failing the whole canvas run.
@@ -17,18 +34,11 @@ func TestQueueMemoryTaskUninitializedQueueReturnsError(t *testing.T) {
engine.SetMessageQueueEngine(natsengine.NewNatsEngine("127.0.0.1", 1))
t.Cleanup(func() { engine.SetMessageQueueEngine(previous) })
- err := queueMemoryTask(
- t.Context(),
- "task-1",
- "mem-1",
- "tenant-1",
- 1,
- MemoryMessage{UserID: "u1", AgentID: "agent-1", SessionID: "s1", UserInput: "hi", AgentResponse: "hello"},
- )
+ err := publishMemoryTaskWakeup(NewMessageQueueTaskPublisher(), "task-1")
if err == nil {
- t.Fatal("queueMemoryTask with uninitialized MQ engine: err = nil, want publish error")
+ t.Fatal("publishMemoryTaskWakeup with uninitialized MQ engine: err = nil, want publish error")
}
if !strings.Contains(err.Error(), "not properly initialized") {
- t.Fatalf("queueMemoryTask err = %v, want engine-not-initialized error", err)
+ t.Fatalf("publishMemoryTaskWakeup err = %v, want engine-not-initialized error", err)
}
}
diff --git a/internal/service/memory_reconciler_test.go b/internal/service/memory_reconciler_test.go
new file mode 100644
index 0000000000..055670ef6a
--- /dev/null
+++ b/internal/service/memory_reconciler_test.go
@@ -0,0 +1,94 @@
+//
+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+package service
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/entity"
+ "ragflow/internal/ingestion/testutil"
+)
+
+// TestReconcileMemoryTasksPublishesDueWakeups verifies the database is the
+// source of recovery work and the broker message carries only durable identity.
+func TestReconcileMemoryTasksPublishesDueWakeups(t *testing.T) {
+ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
+ pinMemoryNow(t, now)
+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ future := now.Add(time.Minute)
+ for _, task := range []*entity.MemoryTask{
+ {TaskID: "due", MemoryID: "memory-1", SourceID: 1, Input: entity.JSONMap{}, State: entity.MemoryTaskStatePending},
+ {TaskID: "future", MemoryID: "memory-1", SourceID: 2, Input: entity.JSONMap{}, State: entity.MemoryTaskStatePending, NextRetryAt: &future},
+ {TaskID: "completed", MemoryID: "memory-1", SourceID: 3, Input: entity.JSONMap{}, State: entity.MemoryTaskStateCompleted},
+ } {
+ if err := db.Create(task).Error; err != nil {
+ t.Fatalf("create memory task %s: %v", task.TaskID, err)
+ }
+ }
+
+ publisher := &recordingTaskPublisher{}
+ svc := NewMemoryMessageService(nil)
+ svc.taskPublisher = publisher
+ if err := svc.ReconcileMemoryTasks(t.Context(), 100); err != nil {
+ t.Fatalf("ReconcileMemoryTasks: %v", err)
+ }
+ if publisher.subject != common.TaskSubject || len(publisher.messages) != 1 {
+ t.Fatalf("published subject/messages = %q/%d", publisher.subject, len(publisher.messages))
+ }
+ message := publisher.messages[0]
+ if message.TaskID != "due" || message.TaskType != common.TaskTypeMemory {
+ t.Fatalf("published wake-up = %+v", message)
+ }
+}
+
+// TestReconcileMemoryTasksStopsPublishingWhenCanceled verifies shutdown stops
+// the current recovery batch before another wake-up is published.
+func TestReconcileMemoryTasksStopsPublishingWhenCanceled(t *testing.T) {
+ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)
+ pinMemoryNow(t, now)
+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})
+ cleanup := testutil.ReplaceDBForTest(t, db)
+ defer cleanup()
+
+ for _, taskID := range []string{"due-1", "due-2"} {
+ if err := db.Create(&entity.MemoryTask{
+ TaskID: taskID, MemoryID: "memory-1", SourceID: 1,
+ Input: entity.JSONMap{}, State: entity.MemoryTaskStatePending,
+ }).Error; err != nil {
+ t.Fatalf("create memory task %s: %v", taskID, err)
+ }
+ }
+
+ ctx, cancel := context.WithCancel(t.Context())
+ publisher := &recordingTaskPublisher{beforeReturn: func(string) { cancel() }}
+ svc := NewMemoryMessageService(nil)
+ svc.taskPublisher = publisher
+ err := svc.ReconcileMemoryTasks(ctx, 100)
+ if !errors.Is(err, context.Canceled) {
+ t.Fatalf("ReconcileMemoryTasks error = %v, want context canceled", err)
+ }
+ if len(publisher.messages) != 1 {
+ t.Fatalf("published messages = %d, want 1 before cancellation", len(publisher.messages))
+ }
+}
diff --git a/internal/service/model_service.go b/internal/service/model_service.go
index 664429d6cc..2070879112 100644
--- a/internal/service/model_service.go
+++ b/internal/service/model_service.go
@@ -35,6 +35,31 @@ import (
"gorm.io/gorm"
)
+// errModelConfigUnavailable marks model configuration failures that cannot be
+// repaired by retrying the same task.
+var errModelConfigUnavailable = errors.New("model configuration unavailable")
+
+// modelInstanceExtra contains the instance fields consumed during model
+// resolution. Other provider-specific fields remain valid and are ignored.
+type modelInstanceExtra struct {
+ Region string `json:"region"`
+ BaseURL string `json:"base_url"`
+}
+
+// decodeModelInstanceExtra reads only the endpoint fields used for model
+// resolution, allowing existing rows to retain provider-specific JSON values.
+func decodeModelInstanceExtra(raw string) (modelInstanceExtra, error) {
+ if strings.TrimSpace(raw) == "" {
+ return modelInstanceExtra{}, nil
+ }
+
+ var extra modelInstanceExtra
+ if err := json.Unmarshal([]byte(raw), &extra); err != nil {
+ return modelInstanceExtra{}, err
+ }
+ return extra, nil
+}
+
// parseModelName parses a composite model name in format "model@instance@provider" or "model@provider"
// Returns modelName, instanceName, providerName separately.
//
@@ -3540,59 +3565,65 @@ func (m *ModelProviderService) GetModelConfigByID(ctx context.Context, userID st
modelEntity, err := m.modelDAO.GetByID(ctx, dao.DB, modelID)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
- return nil, "", nil, 0, fmt.Errorf("tenant model id=%s not found", modelID)
+ return nil, "", nil, 0, fmt.Errorf("%w: tenant model id=%s not found", errModelConfigUnavailable, modelID)
}
return nil, "", nil, 0, err
}
if modelEntity.Status != "active" {
- return nil, "", nil, 0, fmt.Errorf("tenant model id=%s is disabled", modelID)
+ return nil, "", nil, 0, fmt.Errorf("%w: tenant model id=%s is disabled", errModelConfigUnavailable, modelID)
}
if !entity.ModelType(modelEntity.ModelType).Has(modelType) {
- return nil, "", nil, 0, fmt.Errorf("tenant model id=%s cannot be used as %s model", modelID, modelType.String())
+ return nil, "", nil, 0, fmt.Errorf("%w: tenant model id=%s cannot be used as %s model", errModelConfigUnavailable, modelID, modelType.String())
}
providerEntity, err := m.modelProviderDAO.GetByID(ctx, dao.DB, modelEntity.ProviderID)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
- return nil, "", nil, 0, fmt.Errorf("provider id=%s not found for model id=%s", modelEntity.ProviderID, modelID)
+ return nil, "", nil, 0, fmt.Errorf("%w: provider id=%s not found for model id=%s", errModelConfigUnavailable, modelEntity.ProviderID, modelID)
}
return nil, "", nil, 0, err
}
if providerEntity == nil {
- return nil, "", nil, 0, fmt.Errorf("provider id=%s not found for model id=%s", modelEntity.ProviderID, modelID)
+ return nil, "", nil, 0, fmt.Errorf("%w: provider id=%s not found for model id=%s", errModelConfigUnavailable, modelEntity.ProviderID, modelID)
}
allowed, accessErr := m.tenantCanReachProviderTenant(ctx, userID, providerEntity.TenantID)
+
if accessErr != nil {
return nil, "", nil, 0, accessErr
}
if !allowed {
- return nil, "", nil, 0, fmt.Errorf("tenant %s has no access to provider owned by tenant %s", userID, providerEntity.TenantID)
+ return nil, "", nil, 0, fmt.Errorf(
+ "%w: tenant %s has no access to provider owned by tenant %s",
+ errModelConfigUnavailable,
+ userID,
+ providerEntity.TenantID,
+ )
}
instanceEntity, err := m.modelInstanceDAO.GetByID(ctx, dao.DB, modelEntity.InstanceID)
if err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
- return nil, "", nil, 0, fmt.Errorf("instance id=%s not found for model id=%s", modelEntity.InstanceID, modelID)
+ return nil, "", nil, 0, fmt.Errorf("%w: instance id=%s not found for model id=%s", errModelConfigUnavailable, modelEntity.InstanceID, modelID)
}
return nil, "", nil, 0, err
}
apiKey := instanceEntity.APIKey
- var extra map[string]string
- if err := json.Unmarshal([]byte(instanceEntity.Extra), &extra); err != nil {
- return nil, "", nil, 0, err
+ extra, err := decodeModelInstanceExtra(instanceEntity.Extra)
+ if err != nil {
+ return nil, "", nil, 0, fmt.Errorf("%w: decode model instance configuration: %v", errModelConfigUnavailable, err)
}
- region := extra["region"]
- baseURL := extra["base_url"]
+ region := extra.Region
+ baseURL := extra.BaseURL
providerInfo := dao.GetModelProviderManager().FindProvider(providerEntity.ProviderName)
if providerInfo == nil {
- return nil, "", nil, 0, fmt.Errorf("provider %q driver not found", providerEntity.ProviderName)
+ return nil, "", nil, 0, fmt.Errorf("%w: provider %q driver not found", errModelConfigUnavailable, providerEntity.ProviderName)
}
modelDriver, err := newModelDriverForBaseURL(providerInfo.ModelDriver, providerEntity.ProviderName, region, baseURL)
if err != nil {
- return nil, "", nil, 0, err
+ return nil, "", nil, 0, fmt.Errorf("%w: create model driver: %v", errModelConfigUnavailable, err)
}
maxTokens := 0
@@ -3601,7 +3632,7 @@ func (m *ModelProviderService) GetModelConfigByID(ctx context.Context, userID st
}
maxTokens, err = maxTokensFromTenantModelExtra(modelEntity, maxTokens)
if err != nil {
- return nil, "", nil, 0, err
+ return nil, "", nil, 0, fmt.Errorf("%w: read model limits: %v", errModelConfigUnavailable, err)
}
apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: &region, BaseURL: &baseURL}
@@ -3634,7 +3665,7 @@ func defaultModelRefs(tenant *entity.Tenant, modelType entity.ModelType) (string
func (m *ModelProviderService) ResolveModelConfig(ctx context.Context, tenantID string, modelType entity.ModelType, modelRef string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
if strings.TrimSpace(modelRef) == "" {
- return nil, "", nil, 0, fmt.Errorf("model ref is required")
+ return nil, "", nil, 0, fmt.Errorf("%w: model ref is required", errModelConfigUnavailable)
}
if _, err := m.modelDAO.GetByID(ctx, dao.DB, modelRef); err == nil {
return m.GetModelConfigByID(ctx, tenantID, modelType, modelRef)
@@ -4121,7 +4152,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
if modelName == teiModel {
builtinDriver := modelModule.GetBuiltinEmbeddingModel(modelName)
if builtinDriver == nil {
- return nil, "", nil, 0, fmt.Errorf("builtin (TEI) embedding model %q not found", modelName)
+ return nil, "", nil, 0, fmt.Errorf("%w: builtin (TEI) embedding model %q not found", errModelConfigUnavailable, modelName)
}
apiConfig := &modelModule.APIConfig{ApiKey: nil, Region: nil, BaseURL: &teiBaseURL}
return builtinDriver, modelName, apiConfig, 0, nil
@@ -4133,7 +4164,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
if teiPure == teiModel && (teiProvider == "Builtin" || teiProvider == "") {
builtinDriver := modelModule.GetBuiltinEmbeddingModel(teiPure)
if builtinDriver == nil {
- return nil, "", nil, 0, fmt.Errorf("builtin (TEI) embedding model %q not found", teiPure)
+ return nil, "", nil, 0, fmt.Errorf("%w: builtin (TEI) embedding model %q not found", errModelConfigUnavailable, teiPure)
}
apiConfig := &modelModule.APIConfig{ApiKey: nil, Region: nil, BaseURL: &teiBaseURL}
return builtinDriver, teiPure, apiConfig, 0, nil
@@ -4169,7 +4200,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
if pureModelName != "" {
builtinDriver := modelModule.GetBuiltinEmbeddingModel(pureModelName)
if builtinDriver == nil {
- return nil, "", nil, 0, fmt.Errorf("builtin embedding model %q not found", pureModelName)
+ return nil, "", nil, 0, fmt.Errorf("%w: builtin embedding model %q not found", errModelConfigUnavailable, pureModelName)
}
apiKey := ""
region := ""
@@ -4184,33 +4215,41 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
pureModelName, instanceName, providerName, err := parseModelName(modelName)
if err != nil {
- return nil, "", nil, 0, err
+ return nil, "", nil, 0, fmt.Errorf("%w: %v", errModelConfigUnavailable, err)
}
// Direct provider lookup
provider, provErr := m.modelProviderDAO.GetByTenantIDAndProviderName(ctx, dao.DB, tenantID, providerName)
if provErr != nil {
+ if errors.Is(provErr, gorm.ErrRecordNotFound) {
+ return nil, "", nil, 0, fmt.Errorf("%w: provider %q lookup failed: %w", errModelConfigUnavailable, providerName, provErr)
+ }
return nil, "", nil, 0, fmt.Errorf("provider %q lookup failed: %w", providerName, provErr)
}
if provider == nil {
- return nil, "", nil, 0, fmt.Errorf("provider %q not found for model %q", providerName, modelName)
+ return nil, "", nil, 0, fmt.Errorf("%w: provider %q not found for model %q", errModelConfigUnavailable, providerName, modelName)
}
// Direct instance lookup
instance, instErr := m.modelInstanceDAO.GetByProviderIDAndInstanceName(ctx, dao.DB, provider.ID, instanceName)
if instErr != nil {
+ if errors.Is(instErr, gorm.ErrRecordNotFound) {
+ return nil, "", nil, 0, fmt.Errorf("%w: instance %q lookup failed: %w", errModelConfigUnavailable, instanceName, instErr)
+ }
return nil, "", nil, 0, fmt.Errorf("instance %q lookup failed: %w", instanceName, instErr)
}
if instance == nil {
- return nil, "", nil, 0, fmt.Errorf("instance %q not found for model %q", instanceName, modelName)
+ return nil, "", nil, 0, fmt.Errorf("%w: instance %q not found for model %q", errModelConfigUnavailable, instanceName, modelName)
}
// Decode api_key and extra fields from the instance row
apiKey := instance.APIKey
- var extra map[string]string
- _ = json.Unmarshal([]byte(instance.Extra), &extra)
- region := extra["region"]
- baseURL := extra["base_url"]
+ extra, err := decodeModelInstanceExtra(instance.Extra)
+ if err != nil {
+ return nil, "", nil, 0, fmt.Errorf("%w: decode model instance configuration: %v", errModelConfigUnavailable, err)
+ }
+ region := extra.Region
+ baseURL := extra.BaseURL
// Direct model lookup
modelObj, modelErr := m.modelDAO.GetByProviderIDAndInstanceIDAndModelTypeAndModelName(
@@ -4221,16 +4260,16 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
// Happy path: tenant enrolled this model.
// INACTIVE check
if modelObj.Status == "inactive" {
- return nil, "", nil, 0, fmt.Errorf("model %q is disabled", modelName)
+ return nil, "", nil, 0, fmt.Errorf("%w: model %q is disabled", errModelConfigUnavailable, modelName)
}
providerInfo := dao.GetModelProviderManager().FindProvider(providerName)
if providerInfo == nil {
- return nil, "", nil, 0, fmt.Errorf("provider %q driver not found", providerName)
+ return nil, "", nil, 0, fmt.Errorf("%w: provider %q driver not found", errModelConfigUnavailable, providerName)
}
driver, driverErr := newModelDriverForBaseURL(providerInfo.ModelDriver, providerName, region, baseURL)
if driverErr != nil {
- return nil, "", nil, 0, driverErr
+ return nil, "", nil, 0, fmt.Errorf("%w: create model driver: %v", errModelConfigUnavailable, driverErr)
}
maxTokens := 0
if mi, _ := dao.GetModelProviderManager().GetModelByName(providerName, pureModelName); mi != nil {
@@ -4238,7 +4277,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
}
maxTokens, driverErr = maxTokensFromTenantModelExtra(modelObj, maxTokens)
if driverErr != nil {
- return nil, "", nil, 0, driverErr
+ return nil, "", nil, 0, fmt.Errorf("%w: read model limits: %v", errModelConfigUnavailable, driverErr)
}
apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: &region, BaseURL: &baseURL}
return driver, modelObj.ModelName, apiConfig, maxTokens, nil
@@ -4270,7 +4309,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
// casing). The rest of the Go codebase uses these helpers too.
targetProvider := dao.GetModelProviderManager().FindProvider(targetFactoryName)
if targetProvider == nil {
- return nil, "", nil, 0, fmt.Errorf("model provider config not found: %s", providerName)
+ return nil, "", nil, 0, fmt.Errorf("%w: model provider config not found: %s", errModelConfigUnavailable, providerName)
}
var llmInfo *modelModule.Model
for i := range targetProvider.Models {
@@ -4280,11 +4319,11 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co
}
}
if llmInfo == nil {
- return nil, "", nil, 0, fmt.Errorf("model config not found: %s", modelName)
+ return nil, "", nil, 0, fmt.Errorf("%w: model config not found: %s", errModelConfigUnavailable, modelName)
}
driver, driverErr := newModelDriverForBaseURL(targetProvider.ModelDriver, providerName, region, baseURL)
if driverErr != nil {
- return nil, "", nil, 0, driverErr
+ return nil, "", nil, 0, fmt.Errorf("%w: create model driver: %v", errModelConfigUnavailable, driverErr)
}
apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: &region, BaseURL: &baseURL}
maxTokens := maxTokensFromModelInfo(llmInfo, modelType)
diff --git a/internal/service/model_service_test.go b/internal/service/model_service_test.go
index f0959a2d6a..0e3a06bddc 100644
--- a/internal/service/model_service_test.go
+++ b/internal/service/model_service_test.go
@@ -500,6 +500,97 @@ func TestModelProviderServiceGetModelConfigByID(t *testing.T) {
}
}
+func TestModelProviderServiceMissingProviderAndInstancePreserveLookupError(t *testing.T) {
+ db := setupModelProviderServiceTestDB(t)
+ useModelProviderServiceTestDB(t, db)
+ if err := db.Create(&entity.TenantModelProvider{
+ ID: "provider-1",
+ TenantID: "tenant-1",
+ ProviderName: "OpenAI",
+ }).Error; err != nil {
+ t.Fatalf("create provider: %v", err)
+ }
+
+ for _, modelRef := range []string{
+ "unknown@ZHIPU-AI",
+ "unknown@default@OpenAI",
+ } {
+ t.Run(modelRef, func(t *testing.T) {
+ _, _, _, _, err := NewModelProviderService().ResolveModelConfig(
+ t.Context(), "tenant-1", entity.ModelTypeEmbedding, modelRef,
+ )
+ if !errors.Is(err, errModelConfigUnavailable) || !errors.Is(err, gorm.ErrRecordNotFound) {
+ t.Fatalf("ResolveModelConfig() error = %v, want unavailable record-not-found error", err)
+ }
+ if !strings.Contains(err.Error(), "lookup failed: record not found") {
+ t.Fatalf("ResolveModelConfig() error = %q, want lookup failure contract", err)
+ }
+ })
+ }
+}
+
+// TestModelProviderServiceDecodesCompatibleInstanceExtra verifies both model
+// resolution paths accept empty configuration and unrelated typed fields.
+func TestModelProviderServiceDecodesCompatibleInstanceExtra(t *testing.T) {
+ db := setupModelProviderServiceTestDB(t)
+ useModelProviderServiceTestDB(t, db)
+ seedModelProviderServiceScope(t, db)
+
+ svc := NewModelProviderService()
+ resolvers := []struct {
+ name string
+ resolve func() (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error)
+ }{
+ {
+ name: "model ID",
+ resolve: func() (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ return svc.GetModelConfigByID(t.Context(), "user-1", entity.ModelTypeChat, "model-1")
+ },
+ },
+ {
+ name: "provider instance",
+ resolve: func() (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {
+ return svc.ResolveModelConfig(t.Context(), "tenant-1", entity.ModelTypeChat, "gpt-test@default@OpenAI")
+ },
+ },
+ }
+ configs := []struct {
+ name string
+ raw string
+ wantRegion string
+ wantBaseURL string
+ }{
+ {name: "empty", raw: ""},
+ {
+ name: "unrelated typed fields",
+ raw: `{"region":"us-east-1","base_url":"https://models.example.com","enabled":true,"retries":3,"options":{"mode":"custom"}}`,
+ wantRegion: "us-east-1",
+ wantBaseURL: "https://models.example.com",
+ },
+ }
+ for _, config := range configs {
+ t.Run(config.name, func(t *testing.T) {
+ if err := db.Model(&entity.TenantModelInstance{}).
+ Where("id = ?", "instance-1").
+ Update("extra", config.raw).Error; err != nil {
+ t.Fatalf("update instance extra: %v", err)
+ }
+
+ for _, resolver := range resolvers {
+ t.Run(resolver.name, func(t *testing.T) {
+ driver, _, apiConfig, _, err := resolver.resolve()
+ if err != nil {
+ t.Fatalf("resolve model config: %v", err)
+ }
+ if driver == nil || apiConfig == nil || apiConfig.Region == nil || *apiConfig.Region != config.wantRegion || apiConfig.BaseURL == nil || *apiConfig.BaseURL != config.wantBaseURL {
+ t.Fatalf("resolved driver/config = %v/%+v, want region %q and base URL %q", driver, apiConfig, config.wantRegion, config.wantBaseURL)
+ }
+ })
+ }
+ })
+ }
+}
+
func TestMaxTokensFromModelInfo(t *testing.T) {
maxTokens := 4096
maxOutput := 1024