RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(memory): make Go memory extraction tasks durable and resumable (#19512)
taek10521ae263e565a
diff --git a/internal/common/task.go b/internal/common/task.goindex c8c2447cde..91429ba724 100644--- a/internal/common/task.go+++ b/internal/common/task.go@@ -16,8 +16,6 @@package common-import "encoding/json"-const (// TaskSubject is the NATS subject on which ingestion and memory tasks are// published and consumed. Producer and consumer must reference this single@@ -35,20 +33,17 @@ const (// TaskTypeMemory is the async memory-extraction task type. Memory tasks// share the tasks.RAGFLOW subject and the Ingestor's consumer + worker// pool with ingestion tasks; handleAndExecute dispatches them by TaskType.- // The memory-specific payload (message_dict/memory_id/source_id) is- // carried in TaskMessage.Payload.+ // Their TaskMessage is only a wake-up; input lives in memory_task.TaskTypeMemory = "memory")+// TaskMessage is a broker wake-up that identifies one durable task.type TaskMessage struct {TaskID string `json:"task_id" binding:"required"`TaskType string `json:"task_type" binding:"required"`- // Payload carries the task-specific body for non-ingestion task types- // (e.g. the memory extraction payload). It is left empty for ingestion- // tasks and old messages, so existing consumers are unaffected.- Payload json.RawMessage `json:"payload,omitempty"`}+// TaskHandle controls settlement and heartbeat for a received task message.type TaskHandle interface {GetMessage() TaskMessageAck() errordiff --git a/internal/dao/database.go b/internal/dao/database.goindex af0fafc9be..940ed7bfbf 100644--- a/internal/dao/database.go+++ b/internal/dao/database.go@@ -140,6 +140,7 @@ func InitDB(ctx context.Context, migrateDB bool) error {&entity.SyncLogs{},&entity.MCPServer{},&entity.Memory{},+ &entity.MemoryTask{},&entity.Search{},&entity.PipelineOperationLog{},&entity.EvaluationDataset{},@@ -183,7 +184,7 @@ func InitDB(ctx context.Context, migrateDB bool) error {} else {// Ensure Go-exclusive runtime tables exist even if the server starts without --migrateif err = autoMigrateRuntimeModels(ctx, DB); err != nil {- common.Warn("Failed to auto-migrate runtime models", zap.Error(err))+ return fmt.Errorf("failed to auto-migrate runtime models: %w", err)}}// Seed built-in agent templates so the Go backend can serve the@@ -301,10 +302,15 @@ func autoMigrateRuntimeModels(ctx context.Context, db *gorm.DB) error {goRuntimeModels := []interface{}{&entity.IngestionTask{},&entity.IngestionTaskLog{},+ &entity.MemoryTask{},}for _, m := range goRuntimeModels {if err := autoMigrateSafely(ctx, db, m); err != nil {- return fmt.Errorf("failed to auto-migrate runtime model %T: %w", m, err)+ tableName := fmt.Sprintf("%T", m)+ if named, ok := m.(interface{ TableName() string }); ok {+ tableName = named.TableName()+ }+ return fmt.Errorf("failed to auto-migrate runtime table %s: %w", tableName, err)}}return nildiff --git a/internal/dao/database_test.go b/internal/dao/database_test.goindex 204f4bac86..6214b6a4cf 100644--- a/internal/dao/database_test.go+++ b/internal/dao/database_test.go@@ -18,6 +18,7 @@ package daoimport ("context"+ "strings""testing""ragflow/internal/entity"@@ -26,7 +27,7 @@ import ("gorm.io/gorm")-func TestAutoMigrateRuntimeModelsCreatesIngestionTaskTables(t *testing.T) {+func TestAutoMigrateRuntimeModelsCreatesGoRuntimeTables(t *testing.T) {db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})if err != nil {t.Fatalf("open sqlite: %v", err)@@ -39,6 +40,9 @@ func TestAutoMigrateRuntimeModelsCreatesIngestionTaskTables(t *testing.T) {if db.Migrator().HasTable(&entity.IngestionTaskLog{}) {t.Fatal("expected ingestion_task_log to not exist initially")}+ if db.Migrator().HasTable(&entity.MemoryTask{}) {+ t.Fatal("expected memory_task to not exist initially")+ }ctx := context.Background()if err = autoMigrateRuntimeModels(ctx, db); err != nil {@@ -52,9 +56,66 @@ func TestAutoMigrateRuntimeModelsCreatesIngestionTaskTables(t *testing.T) {if !db.Migrator().HasTable(&entity.IngestionTaskLog{}) {t.Fatal("expected ingestion_task_log to exist after autoMigrateRuntimeModels")}+ if !db.Migrator().HasTable(&entity.MemoryTask{}) {+ t.Fatal("expected memory_task to exist after autoMigrateRuntimeModels")+ }+ if !db.Migrator().HasIndex(&entity.MemoryTask{}, "idx_memory_task_due") {+ t.Fatal("expected memory_task due index to exist after autoMigrateRuntimeModels")+ }++ memoryTask := &entity.MemoryTask{+ TaskID: "task-1",+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{+ "user_id": "user-1",+ "user_input": "remember this",+ },+ State: entity.MemoryTaskStatePending,+ Extraction: entity.JSONSlice{+ map[string]interface{}{"message_id": float64(7), "content": "remembered"},+ },+ LastError: "",+ }+ if err = db.Create(memoryTask).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }++ var stored entity.MemoryTask+ if err = db.First(&stored, "task_id = ?", memoryTask.TaskID).Error; err != nil {+ t.Fatalf("load memory task: %v", err)+ }+ if stored.State != entity.MemoryTaskStatePending {+ t.Fatalf("memory task state = %q, want %q", stored.State, entity.MemoryTaskStatePending)+ }+ if got := stored.Input["user_input"]; got != "remember this" {+ t.Fatalf("memory task input user_input = %#v, want %q", got, "remember this")+ }+ if len(stored.Extraction) != 1 {+ t.Fatalf("memory task extraction length = %d, want 1", len(stored.Extraction))+ }// Verify idempotencyif err = autoMigrateRuntimeModels(ctx, db); err != nil {t.Fatalf("second autoMigrateRuntimeModels failed: %v", err)}}++func TestAutoMigrateRuntimeModelsNamesFailingTable(t *testing.T) {+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})+ if err != nil {+ t.Fatalf("open sqlite: %v", err)+ }+ sqlDB, err := db.DB()+ if err != nil {+ t.Fatalf("get sql database: %v", err)+ }+ if err = sqlDB.Close(); err != nil {+ t.Fatalf("close sql database: %v", err)+ }++ err = autoMigrateRuntimeModels(t.Context(), db)+ if err == nil || !strings.Contains(err.Error(), "runtime table ingestion_task") {+ t.Fatalf("autoMigrateRuntimeModels error = %v, want failing table name", err)+ }+}diff --git a/internal/dao/memory.go b/internal/dao/memory.goindex 8b64d935aa..c93bf33cb6 100644--- a/internal/dao/memory.go+++ b/internal/dao/memory.go@@ -273,8 +273,12 @@ func (dao *MemoryDAO) GetWithOwnerNameByID(ctx context.Context, db *gorm.DB, idOwnerName *string `gorm:"column:owner_name"`}- if err := db.WithContext(ctx).Raw(querySQL, id).Scan(&rawResult).Error; err != nil {- return nil, err+ result := db.WithContext(ctx).Raw(querySQL, id).Scan(&rawResult)+ if result.Error != nil {+ return nil, result.Error+ }+ if result.RowsAffected == 0 {+ return nil, gorm.ErrRecordNotFound}return &entity.MemoryListItem{diff --git a/internal/dao/memory_task.go b/internal/dao/memory_task.gonew file mode 100644index 0000000000..523f6d8ce0--- /dev/null+++ b/internal/dao/memory_task.go@@ -0,0 +1,342 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "context"+ "errors"+ "time"++ "ragflow/internal/entity"++ "gorm.io/gorm"+ "gorm.io/gorm/clause"+)++var activeMemoryTaskStates = []entity.MemoryTaskState{+ entity.MemoryTaskStatePending,+ entity.MemoryTaskStateExtracted,+ entity.MemoryTaskStateStored,+}++// MemoryTaskDAO persists durable memory extraction checkpoints and leases.+type MemoryTaskDAO struct{}++// NewMemoryTaskDAO creates a memory task DAO.+func NewMemoryTaskDAO() *MemoryTaskDAO { return &MemoryTaskDAO{} }++// CreateWithTask creates the UI task and its durable execution record in one+// transaction so workers never observe only one side of the pair.+func (d *MemoryTaskDAO) CreateWithTask(ctx context.Context, db *gorm.DB, task *entity.Task, memoryTask *entity.MemoryTask) error {+ if db == nil {+ return errors.New("memory task: nil database")+ }+ if task == nil || memoryTask == nil {+ return errors.New("memory task: task and execution record are required")+ }+ if task.ID == "" || memoryTask.TaskID == "" || task.ID != memoryTask.TaskID {+ return errors.New("memory task: task ids must be non-empty and equal")+ }++ return db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ if err := tx.Create(task).Error; err != nil {+ return err+ }+ return tx.Create(memoryTask).Error+ })+}++// GetByID returns one durable memory task.+func (d *MemoryTaskDAO) GetByID(ctx context.Context, db *gorm.DB, taskID string) (*entity.MemoryTask, error) {+ if db == nil {+ return nil, errors.New("memory task: nil database")+ }+ var task entity.MemoryTask+ if err := db.WithContext(ctx).Where("task_id = ?", taskID).First(&task).Error; err != nil {+ return nil, err+ }+ return &task, nil+}++// Claim acquires an expired or unowned due task and increments its attempt+// count. It returns the current row with acquired=false when the task is not+// due, is terminal, or already has a live lease.+func (d *MemoryTaskDAO) Claim(ctx context.Context, db *gorm.DB, taskID, owner string, now time.Time, leaseTTL time.Duration) (*entity.MemoryTask, bool, error) {+ if db == nil {+ return nil, false, errors.New("memory task: nil database")+ }+ if taskID == "" || owner == "" {+ return nil, false, errors.New("memory task: task id and lease owner are required")+ }+ if leaseTTL <= 0 {+ return nil, false, errors.New("memory task: lease TTL must be positive")+ }++ var (+ task entity.MemoryTask+ acquired bool+ )+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ query := tx.WithContext(ctx)+ if tx.Dialector.Name() != "sqlite" {+ query = query.Clauses(clause.Locking{Strength: "UPDATE"})+ }+ if err := query.Where("task_id = ?", taskID).First(&task).Error; err != nil {+ return err+ }+ if !isActiveMemoryTaskState(task.State) ||+ (task.NextRetryAt != nil && task.NextRetryAt.After(now)) ||+ (task.LeaseExpiresAt != nil && task.LeaseExpiresAt.After(now)) {+ return nil+ }++ expiresAt := now.Add(leaseTTL)+ result := tx.WithContext(ctx).Model(&entity.MemoryTask{}).+ Where("task_id = ?", taskID).+ Updates(map[string]any{+ "attempt_count": gorm.Expr("attempt_count + 1"),+ "lease_owner": owner,+ "lease_expires_at": expiresAt,+ "next_retry_at": nil,+ })+ if result.Error != nil {+ return result.Error+ }+ if result.RowsAffected != 1 {+ return nil+ }+ acquired = true+ return tx.WithContext(ctx).Where("task_id = ?", taskID).First(&task).Error+ })+ if err != nil {+ return nil, false, err+ }+ return &task, acquired, err+}++// RenewLease extends a live lease held by owner.+func (d *MemoryTaskDAO) RenewLease(ctx context.Context, db *gorm.DB, taskID, owner string, now time.Time, leaseTTL time.Duration) (bool, error) {+ if db == nil {+ return false, errors.New("memory task: nil database")+ }+ if taskID == "" || owner == "" || leaseTTL <= 0 {+ return false, errors.New("memory task: task id, lease owner, and positive TTL are required")+ }++ result := db.WithContext(ctx).Model(&entity.MemoryTask{}).+ Where("task_id = ? AND state IN ? AND lease_owner = ? AND lease_expires_at > ?", taskID, activeMemoryTaskStates, owner, now).+ Update("lease_expires_at", now.Add(leaseTTL))+ return result.RowsAffected == 1, result.Error+}++// PersistExtraction advances a leased task from pending to extracted and+// stores the materialized LLM result used by later retries.+func (d *MemoryTaskDAO) PersistExtraction(ctx context.Context, db *gorm.DB, taskID, owner string, now time.Time, extraction entity.JSONSlice) (bool, error) {+ return d.transition(ctx, db, taskID, owner, now, entity.MemoryTaskStatePending, entity.MemoryTaskStateExtracted, map[string]any{+ "extraction": extraction,+ "last_error": "",+ })+}++// MarkStored advances a leased task after its extracted chunks are stored.+func (d *MemoryTaskDAO) MarkStored(ctx context.Context, db *gorm.DB, taskID, owner string, now time.Time) (bool, error) {+ return d.transition(ctx, db, taskID, owner, now, entity.MemoryTaskStateExtracted, entity.MemoryTaskStateStored, map[string]any{+ "last_error": "",+ })+}++// ScheduleRetry records the next execution time and releases the current+// lease. The task remains at its last durable checkpoint.+func (d *MemoryTaskDAO) ScheduleRetry(ctx context.Context, db *gorm.DB, taskID, owner string, now, nextRetryAt time.Time, lastError string) (bool, error) {+ if db == nil {+ return false, errors.New("memory task: nil database")+ }+ if taskID == "" || owner == "" {+ return false, errors.New("memory task: task id and lease owner are required")+ }+ result := db.WithContext(ctx).Model(&entity.MemoryTask{}).+ Where("task_id = ? AND state IN ? AND lease_owner = ? AND lease_expires_at > ?", taskID, activeMemoryTaskStates, owner, now).+ Updates(map[string]any{+ "next_retry_at": nextRetryAt,+ "lease_owner": "",+ "lease_expires_at": nil,+ "last_error": lastError,+ })+ return result.RowsAffected == 1, result.Error+}++// MarkFailed records a terminal failure and projects it onto the generic task+// row when that row still exists.+func (d *MemoryTaskDAO) MarkFailed(ctx context.Context, db *gorm.DB, taskID, owner, progressMsg string, now time.Time) (bool, error) {+ if db == nil {+ return false, errors.New("memory task: nil database")+ }+ if taskID == "" || owner == "" {+ return false, errors.New("memory task: task id and lease owner are required")+ }+ var updated bool+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ result := tx.WithContext(ctx).Model(&entity.MemoryTask{}).+ Where("task_id = ? AND state IN ? AND lease_owner = ? AND lease_expires_at > ?", taskID, activeMemoryTaskStates, owner, now).+ Updates(map[string]any{+ "state": entity.MemoryTaskStateFailed,+ "next_retry_at": nil,+ "lease_owner": "",+ "lease_expires_at": nil,+ "last_error": progressMsg,+ })+ if result.Error != nil || result.RowsAffected != 1 {+ return result.Error+ }+ updated = true+ return tx.WithContext(ctx).Model(&entity.Task{}).Where("id = ?", taskID).Updates(map[string]any{+ "progress": -1,+ "progress_msg": progressMsg,+ }).Error+ })+ if err != nil {+ return false, err+ }+ return updated, nil+}++// MarkUnclaimedFailed records a terminal failure before a newly created task+// has been published or claimed, and updates its UI projection atomically.+func (d *MemoryTaskDAO) MarkUnclaimedFailed(ctx context.Context, db *gorm.DB, taskID, progressMsg string) (bool, error) {+ if db == nil {+ return false, errors.New("memory task: nil database")+ }+ if taskID == "" {+ return false, errors.New("memory task: task id is required")+ }++ var updated bool+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ result := tx.WithContext(ctx).Model(&entity.MemoryTask{}).+ Where("task_id = ? AND state = ? AND lease_owner = '' AND lease_expires_at IS NULL", taskID, entity.MemoryTaskStatePending).+ Updates(map[string]any{+ "state": entity.MemoryTaskStateFailed,+ "next_retry_at": nil,+ "last_error": progressMsg,+ })+ if result.Error != nil {+ return result.Error+ }+ if result.RowsAffected != 1 {+ return nil+ }++ if err := tx.WithContext(ctx).Model(&entity.Task{}).Where("id = ?", taskID).Updates(map[string]any{+ "progress": -1,+ "progress_msg": progressMsg,+ }).Error; err != nil {+ return err+ }+ updated = true+ return nil+ })+ if err != nil {+ return false, err+ }+ return updated, nil+}++// Complete atomically advances a leased stored task to completed and updates+// the generic task progress projection when it still exists.+func (d *MemoryTaskDAO) Complete(ctx context.Context, db *gorm.DB, taskID, owner, progressMsg string, now time.Time) (bool, error) {+ if db == nil {+ return false, errors.New("memory task: nil database")+ }+ if taskID == "" || owner == "" {+ return false, errors.New("memory task: task id and lease owner are required")+ }+ var completed bool+ err := db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {+ result := tx.WithContext(ctx).Model(&entity.MemoryTask{}).+ Where("task_id = ? AND state = ? AND lease_owner = ? AND lease_expires_at > ?", taskID, entity.MemoryTaskStateStored, owner, now).+ Updates(map[string]any{+ "state": entity.MemoryTaskStateCompleted,+ "next_retry_at": nil,+ "lease_owner": "",+ "lease_expires_at": nil,+ "last_error": "",+ })+ if result.Error != nil {+ return result.Error+ }+ if result.RowsAffected != 1 {+ return nil+ }++ progress := tx.WithContext(ctx).Model(&entity.Task{}).Where("id = ?", taskID).Updates(map[string]any{+ "progress": 1.0,+ "progress_msg": progressMsg,+ })+ if progress.Error != nil {+ return progress.Error+ }+ completed = true+ return nil+ })+ if err != nil {+ return false, err+ }+ return completed, nil+}++// ListDue returns non-terminal tasks whose retry time and lease allow another+// wake-up to be published.+func (d *MemoryTaskDAO) ListDue(ctx context.Context, db *gorm.DB, now time.Time, limit int) ([]*entity.MemoryTask, error) {+ if db == nil {+ return nil, errors.New("memory task: nil database")+ }+ if limit <= 0 {+ return nil, errors.New("memory task: positive list limit is required")+ }++ var tasks []*entity.MemoryTask+ err := db.WithContext(ctx).+ Where("state IN ?", activeMemoryTaskStates).+ Where("next_retry_at IS NULL OR next_retry_at <= ?", now).+ Where("lease_expires_at IS NULL OR lease_expires_at <= ?", now).+ Order("next_retry_at ASC, create_time ASC, task_id ASC").+ Limit(limit).+ Find(&tasks).Error+ return tasks, err+}++// transition applies a lease-guarded checkpoint change.+func (d *MemoryTaskDAO) transition(ctx context.Context, db *gorm.DB, taskID, owner string, now time.Time, from, to entity.MemoryTaskState, updates map[string]any) (bool, error) {+ if db == nil {+ return false, errors.New("memory task: nil database")+ }+ if taskID == "" || owner == "" {+ return false, errors.New("memory task: task id and lease owner are required")+ }+ updates["state"] = to+ result := db.WithContext(ctx).Model(&entity.MemoryTask{}).+ Where("task_id = ? AND state = ? AND lease_owner = ? AND lease_expires_at > ?", taskID, from, owner, now).+ Updates(updates)+ return result.RowsAffected == 1, result.Error+}++// isActiveMemoryTaskState reports whether a task can still be executed.+func isActiveMemoryTaskState(state entity.MemoryTaskState) bool {+ return state == entity.MemoryTaskStatePending ||+ state == entity.MemoryTaskStateExtracted ||+ state == entity.MemoryTaskStateStored+}diff --git a/internal/dao/memory_task_test.go b/internal/dao/memory_task_test.gonew file mode 100644index 0000000000..b40503a445--- /dev/null+++ b/internal/dao/memory_task_test.go@@ -0,0 +1,342 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "testing"+ "time"++ "ragflow/internal/entity"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"+)++// setupMemoryTaskTestDB creates an isolated SQLite store for DAO tests.+func setupMemoryTaskTestDB(t *testing.T) *gorm.DB {+ t.Helper()+ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{})+ if err != nil {+ t.Fatalf("open sqlite: %v", err)+ }+ if err = db.AutoMigrate(&entity.Task{}, &entity.MemoryTask{}); err != nil {+ t.Fatalf("auto-migrate memory tasks: %v", err)+ }+ sqlDB, err := db.DB()+ if err != nil {+ t.Fatalf("get sql database: %v", err)+ }+ sqlDB.SetMaxOpenConns(1)+ return db+}++// newMemoryTaskPair returns matching UI and execution records.+func newMemoryTaskPair(taskID string) (*entity.Task, *entity.MemoryTask) {+ progressMsg := ""+ task := &entity.Task{+ ID: taskID,+ DocID: "memory-1",+ TaskType: "memory",+ ProgressMsg: &progressMsg,+ }+ memoryTask := &entity.MemoryTask{+ TaskID: taskID,+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{+ "user_id": "user-1",+ "user_input": "remember this",+ },+ State: entity.MemoryTaskStatePending,+ LastError: "",+ }+ return task, memoryTask+}++// TestMemoryTaskDAOCreateWithTaskIsAtomic verifies a failed execution-record+// insert rolls back the generic task insert.+func TestMemoryTaskDAOCreateWithTaskIsAtomic(t *testing.T) {+ db := setupMemoryTaskTestDB(t)+ dao := NewMemoryTaskDAO()++ _, existing := newMemoryTaskPair("task-1")+ if err := db.Create(existing).Error; err != nil {+ t.Fatalf("seed memory task: %v", err)+ }+ task, memoryTask := newMemoryTaskPair("task-1")+ if err := dao.CreateWithTask(t.Context(), db, task, memoryTask); err == nil {+ t.Fatal("CreateWithTask error = nil, want duplicate-key error")+ }++ var taskCount int64+ if err := db.Model(&entity.Task{}).Where("id = ?", task.ID).Count(&taskCount).Error; err != nil {+ t.Fatalf("count generic tasks: %v", err)+ }+ if taskCount != 0 {+ t.Fatalf("generic task count = %d, want transaction rollback", taskCount)+ }+}++// TestMemoryTaskDAOClaimRenewAndRetry verifies lease exclusion, renewal, and+// retry-time admission.+func TestMemoryTaskDAOClaimRenewAndRetry(t *testing.T) {+ db := setupMemoryTaskTestDB(t)+ dao := NewMemoryTaskDAO()+ task, memoryTask := newMemoryTaskPair("task-1")+ if err := dao.CreateWithTask(t.Context(), db, task, memoryTask); err != nil {+ t.Fatalf("CreateWithTask: %v", err)+ }++ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)+ claimed, acquired, err := dao.Claim(t.Context(), db, task.ID, "worker-1", now, time.Minute)+ if err != nil || !acquired {+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)+ }+ if claimed.AttemptCount != 1 || claimed.LeaseOwner != "worker-1" {+ t.Fatalf("claimed task = %+v, want attempt 1 owned by worker-1", claimed)+ }++ if _, acquired, err = dao.Claim(t.Context(), db, task.ID, "worker-2", now.Add(30*time.Second), time.Minute); err != nil || acquired {+ t.Fatalf("duplicate Claim acquired=%v err=%v, want live lease rejection", acquired, err)+ }+ if renewed, renewErr := dao.RenewLease(t.Context(), db, task.ID, "worker-1", now.Add(30*time.Second), time.Minute); renewErr != nil || !renewed {+ t.Fatalf("RenewLease renewed=%v err=%v, want renewed", renewed, renewErr)+ }++ retryAt := now.Add(2 * time.Minute)+ if scheduled, scheduleErr := dao.ScheduleRetry(t.Context(), db, task.ID, "worker-1", now.Add(30*time.Second), retryAt, "temporary failure"); scheduleErr != nil || !scheduled {+ t.Fatalf("ScheduleRetry scheduled=%v err=%v, want scheduled", scheduled, scheduleErr)+ }+ if _, acquired, err = dao.Claim(t.Context(), db, task.ID, "worker-2", retryAt.Add(-time.Second), time.Minute); err != nil || acquired {+ t.Fatalf("early retry Claim acquired=%v err=%v, want not due", acquired, err)+ }+ claimed, acquired, err = dao.Claim(t.Context(), db, task.ID, "worker-2", retryAt, time.Minute)+ if err != nil || !acquired {+ t.Fatalf("due retry Claim acquired=%v err=%v, want acquired", acquired, err)+ }+ if claimed.AttemptCount != 2 || claimed.LeaseOwner != "worker-2" {+ t.Fatalf("retried task = %+v, want attempt 2 owned by worker-2", claimed)+ }+}++// TestMemoryTaskDAOClaimAfterLeaseExpiry verifies a crashed worker cannot+// strand a task after its durable lease expires.+func TestMemoryTaskDAOClaimAfterLeaseExpiry(t *testing.T) {+ db := setupMemoryTaskTestDB(t)+ dao := NewMemoryTaskDAO()+ task, memoryTask := newMemoryTaskPair("task-expired-lease")+ if err := dao.CreateWithTask(t.Context(), db, task, memoryTask); err != nil {+ t.Fatalf("CreateWithTask: %v", err)+ }++ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)+ leaseTTL := time.Minute+ if _, acquired, err := dao.Claim(t.Context(), db, task.ID, "worker-1", now, leaseTTL); err != nil || !acquired {+ t.Fatalf("first Claim acquired=%v err=%v, want acquired", acquired, err)+ }+ if _, acquired, err := dao.Claim(t.Context(), db, task.ID, "worker-2", now.Add(leaseTTL-time.Second), leaseTTL); err != nil || acquired {+ t.Fatalf("live-lease Claim acquired=%v err=%v, want rejected", acquired, err)+ }+ claimed, acquired, err := dao.Claim(t.Context(), db, task.ID, "worker-2", now.Add(leaseTTL), leaseTTL)+ if err != nil || !acquired {+ t.Fatalf("expired-lease Claim acquired=%v err=%v, want acquired", acquired, err)+ }+ if claimed.LeaseOwner != "worker-2" || claimed.AttemptCount != 2 {+ t.Fatalf("reclaimed task owner/attempt = %q/%d, want worker-2/2", claimed.LeaseOwner, claimed.AttemptCount)+ }+}++// TestMemoryTaskDAOExpiredLeaseCannotRecordFailure verifies a stale worker+// cannot schedule a retry or mark a task failed after its lease expires.+func TestMemoryTaskDAOExpiredLeaseCannotRecordFailure(t *testing.T) {+ db := setupMemoryTaskTestDB(t)+ dao := NewMemoryTaskDAO()+ task, memoryTask := newMemoryTaskPair("task-expired-failure")+ if err := dao.CreateWithTask(t.Context(), db, task, memoryTask); err != nil {+ t.Fatalf("CreateWithTask: %v", err)+ }++ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)+ leaseTTL := time.Minute+ if _, acquired, err := dao.Claim(t.Context(), db, task.ID, "worker-1", now, leaseTTL); err != nil || !acquired {+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)+ }+ expiredAt := now.Add(leaseTTL)+ if scheduled, err := dao.ScheduleRetry(t.Context(), db, task.ID, "worker-1", expiredAt, expiredAt.Add(time.Minute), "temporary failure"); err != nil || scheduled {+ t.Fatalf("ScheduleRetry scheduled=%v err=%v, want expired lease rejection", scheduled, err)+ }+ if failed, err := dao.MarkFailed(t.Context(), db, task.ID, "worker-1", "permanent failure", expiredAt); err != nil || failed {+ t.Fatalf("MarkFailed failed=%v err=%v, want expired lease rejection", failed, err)+ }++ stored, err := dao.GetByID(t.Context(), db, task.ID)+ if err != nil {+ t.Fatalf("GetByID: %v", err)+ }+ if stored.State != entity.MemoryTaskStatePending || stored.LeaseOwner != "worker-1" || stored.NextRetryAt != nil || stored.LastError != "" {+ t.Fatalf("memory task changed after expired-lease updates: %+v", stored)+ }+ var genericTask entity.Task+ if err = db.First(&genericTask, "id = ?", task.ID).Error; err != nil {+ t.Fatalf("load generic task: %v", err)+ }+ if genericTask.Progress != 0 {+ t.Fatalf("generic task progress = %v, want unchanged", genericTask.Progress)+ }+}++// TestMemoryTaskDAOCheckpointAndComplete verifies the complete durable state+// sequence and final UI progress projection.+func TestMemoryTaskDAOCheckpointAndComplete(t *testing.T) {+ db := setupMemoryTaskTestDB(t)+ dao := NewMemoryTaskDAO()+ task, memoryTask := newMemoryTaskPair("task-1")+ if err := dao.CreateWithTask(t.Context(), db, task, memoryTask); err != nil {+ t.Fatalf("CreateWithTask: %v", err)+ }++ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)+ if _, acquired, err := dao.Claim(t.Context(), db, task.ID, "worker-1", now, time.Minute); err != nil || !acquired {+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)+ }+ extraction := entity.JSONSlice{+ map[string]interface{}{"message_id": float64(7), "content": "remembered"},+ }+ if updated, err := dao.PersistExtraction(t.Context(), db, task.ID, "worker-1", now, extraction); err != nil || !updated {+ t.Fatalf("PersistExtraction updated=%v err=%v, want updated", updated, err)+ }+ if updated, err := dao.MarkStored(t.Context(), db, task.ID, "worker-1", now); err != nil || !updated {+ t.Fatalf("MarkStored updated=%v err=%v, want updated", updated, err)+ }+ if completed, err := dao.Complete(t.Context(), db, task.ID, "worker-1", "complete", now); err != nil || !completed {+ t.Fatalf("Complete completed=%v err=%v, want completed", completed, err)+ }++ stored, err := dao.GetByID(t.Context(), db, task.ID)+ if err != nil {+ t.Fatalf("GetByID: %v", err)+ }+ if stored.State != entity.MemoryTaskStateCompleted || stored.LeaseOwner != "" || stored.LeaseExpiresAt != nil {+ t.Fatalf("completed memory task = %+v", stored)+ }+ var genericTask entity.Task+ if err = db.First(&genericTask, "id = ?", task.ID).Error; err != nil {+ t.Fatalf("load generic task: %v", err)+ }+ if genericTask.Progress != 1 || genericTask.ProgressMsg == nil || *genericTask.ProgressMsg != "complete" {+ t.Fatalf("generic task progress = %v message=%v, want 1/complete", genericTask.Progress, genericTask.ProgressMsg)+ }+}++// TestMemoryTaskDAOCompleteRollsBackWhenProgressUpdateFails verifies the+// stored checkpoint remains retryable when the UI projection cannot be+// committed in the same transaction.+func TestMemoryTaskDAOCompleteRollsBackWhenProgressUpdateFails(t *testing.T) {+ db := setupMemoryTaskTestDB(t)+ dao := NewMemoryTaskDAO()+ task, memoryTask := newMemoryTaskPair("task-completion-failure")+ memoryTask.State = entity.MemoryTaskStateStored+ memoryTask.LeaseOwner = "worker-1"+ expiresAt := time.Date(2026, 9, 10, 12, 1, 0, 0, time.UTC)+ memoryTask.LeaseExpiresAt = &expiresAt+ if err := dao.CreateWithTask(t.Context(), db, task, memoryTask); err != nil {+ t.Fatalf("CreateWithTask: %v", err)+ }+ if err := db.Exec(`+ CREATE TRIGGER fail_memory_task_progress_update+ BEFORE UPDATE ON task+ BEGIN+ SELECT RAISE(FAIL, 'forced progress update failure');+ END+ `).Error; err != nil {+ t.Fatalf("create update trigger: %v", err)+ }++ completed, err := dao.Complete(t.Context(), db, task.ID, "worker-1", "complete", expiresAt.Add(-time.Second))+ if err == nil || completed {+ t.Fatalf("Complete completed=%v err=%v, want transaction failure", completed, err)+ }+ stored, getErr := dao.GetByID(t.Context(), db, task.ID)+ if getErr != nil {+ t.Fatalf("GetByID: %v", getErr)+ }+ if stored.State != entity.MemoryTaskStateStored || stored.LeaseOwner != "worker-1" || stored.LeaseExpiresAt == nil {+ t.Fatalf("memory task after rollback = %+v, want stored checkpoint and original lease", stored)+ }+ var genericTask entity.Task+ if getErr = db.First(&genericTask, "id = ?", task.ID).Error; getErr != nil {+ t.Fatalf("load generic task: %v", getErr)+ }+ if genericTask.Progress != 0 {+ t.Fatalf("generic task progress = %v, want unchanged", genericTask.Progress)+ }+}++// TestMemoryTaskDAOCompleteWithoutGenericTask verifies a missing optional UI+// projection does not roll back the authoritative durable completion state.+func TestMemoryTaskDAOCompleteWithoutGenericTask(t *testing.T) {+ db := setupMemoryTaskTestDB(t)+ dao := NewMemoryTaskDAO()+ _, memoryTask := newMemoryTaskPair("task-1")+ memoryTask.State = entity.MemoryTaskStateStored+ memoryTask.LeaseOwner = "worker-1"+ expiresAt := time.Date(2026, 9, 10, 12, 1, 0, 0, time.UTC)+ memoryTask.LeaseExpiresAt = &expiresAt+ if err := db.Create(memoryTask).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }++ completed, err := dao.Complete(t.Context(), db, memoryTask.TaskID, "worker-1", "complete", expiresAt.Add(-time.Second))+ if err != nil || !completed {+ t.Fatalf("Complete completed=%v err=%v, want completed", completed, err)+ }+ stored, getErr := dao.GetByID(t.Context(), db, memoryTask.TaskID)+ if getErr != nil {+ t.Fatalf("GetByID: %v", getErr)+ }+ if stored.State != entity.MemoryTaskStateCompleted || stored.LeaseOwner != "" || stored.LeaseExpiresAt != nil {+ t.Fatalf("completed memory task = %+v", stored)+ }+}++// TestMemoryTaskDAOListDueExcludesFutureLeasedAndTerminalTasks verifies the+// reconciler query returns only executable rows.+func TestMemoryTaskDAOListDueExcludesFutureLeasedAndTerminalTasks(t *testing.T) {+ db := setupMemoryTaskTestDB(t)+ dao := NewMemoryTaskDAO()+ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)+ future := now.Add(time.Minute)++ for _, task := range []*entity.MemoryTask{+ {TaskID: "due", MemoryID: "memory-1", SourceID: 1, Input: entity.JSONMap{}, State: entity.MemoryTaskStatePending, LastError: ""},+ {TaskID: "future", MemoryID: "memory-1", SourceID: 2, Input: entity.JSONMap{}, State: entity.MemoryTaskStatePending, NextRetryAt: &future, LastError: ""},+ {TaskID: "leased", MemoryID: "memory-1", SourceID: 3, Input: entity.JSONMap{}, State: entity.MemoryTaskStateExtracted, LeaseOwner: "worker-1", LeaseExpiresAt: &future, LastError: ""},+ {TaskID: "completed", MemoryID: "memory-1", SourceID: 4, Input: entity.JSONMap{}, State: entity.MemoryTaskStateCompleted, LastError: ""},+ } {+ if err := db.Create(task).Error; err != nil {+ t.Fatalf("create task %s: %v", task.TaskID, err)+ }+ }++ tasks, err := dao.ListDue(t.Context(), db, now, 10)+ if err != nil {+ t.Fatalf("ListDue: %v", err)+ }+ if len(tasks) != 1 || tasks[0].TaskID != "due" {+ t.Fatalf("ListDue = %+v, want only due task", tasks)+ }+}diff --git a/internal/dao/task.go b/internal/dao/task.goindex 18624897e6..bc0251a06f 100644--- a/internal/dao/task.go+++ b/internal/dao/task.go@@ -18,6 +18,8 @@ package daoimport ("context"++ "ragflow/internal/common""ragflow/internal/entity""gorm.io/gorm"@@ -62,6 +64,18 @@ func (dao *TaskDAO) UpdateProgress(ctx context.Context, db *gorm.DB, id string,}).Error}+// MarkActiveMemoryTaskFailed marks an unfinished memory task's UI projection+// as failed without overwriting a task that completed or failed concurrently.+func (dao *TaskDAO) MarkActiveMemoryTaskFailed(ctx context.Context, db *gorm.DB, id, progressMsg string) (bool, error) {+ result := db.WithContext(ctx).Model(&entity.Task{}).+ Where("id = ? AND task_type = ? AND progress >= 0 AND progress < 1", id, common.TaskTypeMemory).+ Updates(map[string]any{+ "progress": -1,+ "progress_msg": progressMsg,+ })+ return result.RowsAffected == 1, result.Error+}+// DeleteIngestionTasksByDocIDs deletes ingestion tasks by document IDs (hard delete)func (dao *TaskDAO) DeleteIngestionTasksByDocIDs(ctx context.Context, db *gorm.DB, docIDs []string) (int64, error) {if len(docIDs) == 0 {diff --git a/internal/entity/memory_task.go b/internal/entity/memory_task.gonew file mode 100644index 0000000000..e7650f6623--- /dev/null+++ b/internal/entity/memory_task.go@@ -0,0 +1,51 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package entity++import "time"++// MemoryTaskState is a durable checkpoint in memory extraction.+type MemoryTaskState string++const (+ MemoryTaskStatePending MemoryTaskState = "pending"+ MemoryTaskStateExtracted MemoryTaskState = "extracted"+ MemoryTaskStateStored MemoryTaskState = "stored"+ MemoryTaskStateCompleted MemoryTaskState = "completed"+ MemoryTaskStateFailed MemoryTaskState = "failed"+)++// MemoryTask stores the authoritative execution state for an asynchronous+// memory extraction task. Task.Progress remains a UI projection and is not an+// execution checkpoint.+type MemoryTask struct {+ TaskID string `gorm:"column:task_id;primaryKey;size:32" json:"task_id"`+ MemoryID string `gorm:"column:memory_id;size:32;not null;index" json:"memory_id"`+ SourceID int64 `gorm:"column:source_id;not null" json:"source_id"`+ Input JSONMap `gorm:"column:input;type:longtext;not null" json:"input"`+ State MemoryTaskState `gorm:"column:state;size:16;not null;default:'pending';index:idx_memory_task_due,priority:1" json:"state"`+ Extraction JSONSlice `gorm:"column:extraction;type:longtext" json:"extraction,omitempty"`+ NextRetryAt *time.Time `gorm:"column:next_retry_at;default:null;index:idx_memory_task_due,priority:2" json:"next_retry_at,omitempty"`+ AttemptCount int `gorm:"column:attempt_count;not null;default:0" json:"attempt_count"`+ LeaseOwner string `gorm:"column:lease_owner;size:128;not null;default:''" json:"lease_owner"`+ LeaseExpiresAt *time.Time `gorm:"column:lease_expires_at;default:null;index" json:"lease_expires_at,omitempty"`+ LastError string `gorm:"column:last_error;type:text;not null" json:"last_error"`+ BaseModel+}++// TableName returns the durable memory task table name.+func (MemoryTask) TableName() string { return "memory_task" }diff --git a/internal/ingestion/service/execute_memory_task_panic_test.go b/internal/ingestion/service/execute_memory_task_panic_test.goindex 45cb9ac82d..2cc0065d2a 100644--- a/internal/ingestion/service/execute_memory_task_panic_test.go+++ b/internal/ingestion/service/execute_memory_task_panic_test.go@@ -24,7 +24,7 @@ import (// so it does not depend on a live DB or a real MemoryMessageService. The contract// asserted:// - executeMemoryTask returns normally (no panic propagates to the caller/worker)-// - the MQ message is Nacked so the broker redelivers it (never silently dropped)+// - the MQ message is left unsettled for durable recoveryfunc TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {_ = testutil.SetupTestDB(t) // ensure DB helpers are initialized; not exercised here@@ -34,15 +34,12 @@ func TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))// Inject a panicking memory runner through the same seam used in production// (defaultRunMemoryTask calls memorySvc.HandleSaveToMemoryTask).- ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {+ ingestor.runMemoryTask = func(_ context.Context, _, _ string) (servicepkg.MemoryTaskDisposition, error) {panic("simulated memory extraction panic")}handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-1", TaskType: common.TaskTypeMemory}}- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-panic-1", map[string]any{- "memory_id": "mem-p", "source_id": 1,- "message_dict": map[string]any{"user_id": "u", "agent_id": "a", "session_id": "s"},- }, handle)+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(context.Background(), "mem-panic-1", handle)// If the panic is not recovered, this call itself will panic and the test// fails. We also guard with a recover here only to convert a propagated@@ -56,8 +53,8 @@ func TestExecuteMemoryTask_PanicDoesNotPropagate(t *testing.T) {ingestor.executeMemoryTask(context.Background(), taskCtx)}()- if handle.nacks.Load() != 1 || handle.acks.Load() != 0 {- t.Fatalf("panicking memory task: expected 0 Ack/1 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ if handle.nacks.Load() != 0 || handle.acks.Load() != 0 {+ t.Fatalf("panicking memory task: expected unsettled handle, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())}}@@ -73,7 +70,7 @@ func TestHandleAndExecute_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {ingestor := NewIngestor("test", 1, []string{"pdf"})ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))- ingestor.runMemoryTask = func(_ context.Context, _ string, _ map[string]any) error {+ ingestor.runMemoryTask = func(_ context.Context, _, _ string) (servicepkg.MemoryTaskDisposition, error) {panic("simulated memory extraction panic")}ingestor.runDocumentTask = func(ctx context.Context, _ *entity.IngestionTask) error {@@ -83,7 +80,6 @@ func TestHandleAndExecute_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {memHandle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-panic-2",TaskType: common.TaskTypeMemory,- Payload: []byte(`{"memory_id":"mem-p2","source_id":1,"message_dict":{"user_id":"u","agent_id":"a","session_id":"s"}}`),}}ingestor.handleAndExecute(memHandle)@@ -91,8 +87,8 @@ func TestHandleAndExecute_SurvivesMemoryPanicAndKeepsServing(t *testing.T) {docHandle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: taskID, TaskType: common.TaskTypeIngestionTask}}ingestor.handleAndExecute(docHandle)- if memHandle.nacks.Load() != 1 || memHandle.acks.Load() != 0 {- t.Fatalf("poison memory task: expected 0 Ack/1 Nack, got acks=%d nacks=%d", memHandle.acks.Load(), memHandle.nacks.Load())+ if memHandle.nacks.Load() != 0 || memHandle.acks.Load() != 0 {+ t.Fatalf("poison memory task: expected unsettled handle, got acks=%d nacks=%d", memHandle.acks.Load(), memHandle.nacks.Load())}if docHandle.acks.Load() != 1 || docHandle.nacks.Load() != 0 {t.Fatalf("document task after memory panic: expected 1 Ack/0 Nack (worker survived), got acks=%d nacks=%d", docHandle.acks.Load(), docHandle.nacks.Load())diff --git a/internal/ingestion/service/handle_and_execute_test.go b/internal/ingestion/service/handle_and_execute_test.goindex 4df6dd4944..9a7fbfbb5b 100644--- a/internal/ingestion/service/handle_and_execute_test.go+++ b/internal/ingestion/service/handle_and_execute_test.go@@ -18,6 +18,7 @@ package serviceimport ("context"+ "errors""testing""time"@@ -27,28 +28,30 @@ import (servicepkg "ragflow/internal/service")-// TestHandleAndExecute_MalformedMemoryPayloadAcks verifies that a memory task with-// an unparseable payload is acked and skipped without executing the runner.-func TestHandleAndExecute_MalformedMemoryPayloadAcks(t *testing.T) {+// TestHandleAndExecute_MemoryTaskIDInvokesRunner verifies memory deliveries+// need only the task id because execution input lives in the database.+func TestHandleAndExecute_MemoryTaskIDInvokesRunner(t *testing.T) {ingestor := newUnitIngestor("test-mem-malformed", 1, nil)ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))runnerCalled := false- ingestor.runMemoryTask = func(ctx context.Context, taskID string, payload map[string]any) error {+ ingestor.runMemoryTask = func(ctx context.Context, taskID, leaseOwner string) (servicepkg.MemoryTaskDisposition, error) {runnerCalled = true- return nil+ if taskID != "mem-bad-payload" || leaseOwner == "" {+ t.Fatalf("runner identity = %q/%q", taskID, leaseOwner)+ }+ return servicepkg.MemoryTaskAcknowledge, nil}handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-bad-payload",TaskType: common.TaskTypeMemory,- Payload: []byte(`{invalid-json`),}}ingestor.handleAndExecute(handle)- if runnerCalled {- t.Fatal("expected runMemoryTask to not be called for malformed payload")+ if !runnerCalled {+ t.Fatal("expected runMemoryTask to be called with task id only")}if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {t.Fatalf("expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())@@ -64,7 +67,6 @@ func TestHandleAndExecute_MemoryExtractorDisabledAcks(t *testing.T) {handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "mem-disabled",TaskType: common.TaskTypeMemory,- Payload: []byte(`{"memory_id":"m1"}`),}}ingestor.handleAndExecute(handle)@@ -83,7 +85,6 @@ func TestHandleAndExecute_MemoryEmptyTaskIDAcks(t *testing.T) {handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "",TaskType: common.TaskTypeMemory,- Payload: []byte(`{"memory_id":"m1"}`),}}ingestor.handleAndExecute(handle)@@ -277,33 +278,48 @@ func TestHandleAndExecute_DocumentDuplicateClaimRenewsLease(t *testing.T) {}}-// TestHandleAndExecute_MemoryDuplicateClaimRenewsLease verifies duplicate delivery-// protection for memory extraction tasks.-func TestHandleAndExecute_MemoryDuplicateClaimRenewsLease(t *testing.T) {+// TestHandleAndExecute_MemoryUnsettledDispositionDefersToDurableRecovery+// verifies DB lease admission controls settlement for duplicate or failed work.+func TestHandleAndExecute_MemoryUnsettledDispositionDefersToDurableRecovery(t *testing.T) {ingestor := newUnitIngestor("test-mem-dup", 1, nil)ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))-- taskID := "mem-dup-1"- if !ingestor.claimTask(taskID) {- t.Fatal("first claim should succeed")+ ingestor.runMemoryTask = func(context.Context, string, string) (servicepkg.MemoryTaskDisposition, error) {+ return servicepkg.MemoryTaskLeaveUnsettled, nil}handle := &fakeTaskHandle{msg: common.TaskMessage{- TaskID: taskID,+ TaskID: "mem-dup-1",TaskType: common.TaskTypeMemory,- Payload: []byte(`{"memory_id":"m1","source_id":1,"message_dict":{"user_id":"u"}}`),}}ingestor.handleAndExecute(handle)- if handle.inProgress.Load() != 1 {- t.Fatalf("expected InProgress = 1 for duplicate memory delivery, got %d", handle.inProgress.Load())- }if handle.acks.Load() != 0 || handle.nacks.Load() != 0 {t.Fatalf("expected 0 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())}}+// TestHandleAndExecute_MemoryPersistedRetryAcks verifies an execution error is+// still acknowledged after the service durably schedules its retry.+func TestHandleAndExecute_MemoryPersistedRetryAcks(t *testing.T) {+ ingestor := newUnitIngestor("test-mem-retry", 1, nil)+ ingestor.SetMemoryMessageService(servicepkg.NewMemoryMessageService(servicepkg.NewMemoryService()))+ ingestor.runMemoryTask = func(context.Context, string, string) (servicepkg.MemoryTaskDisposition, error) {+ return servicepkg.MemoryTaskAcknowledge, errors.New("retry scheduled")+ }++ handle := &fakeTaskHandle{msg: common.TaskMessage{+ TaskID: "mem-retry-1",+ TaskType: common.TaskTypeMemory,+ }}++ ingestor.handleAndExecute(handle)++ if handle.acks.Load() != 1 || handle.nacks.Load() != 0 {+ t.Fatalf("expected 1 Ack/0 Nack, got acks=%d nacks=%d", handle.acks.Load(), handle.nacks.Load())+ }+}+// TestHandleAndExecute_SlowAdmissionHeartbeatsUnderLeaseProtection verifies that// when admission (StartRunning) is slow (e.g. database lag exceeding ack deadline),// the heartbeat started upon receiving the handle renews the broker lease beforediff --git a/internal/ingestion/service/ingestion_service.go b/internal/ingestion/service/ingestion_service.goindex 4547454a4c..53e18fdad8 100644--- a/internal/ingestion/service/ingestion_service.go+++ b/internal/ingestion/service/ingestion_service.go@@ -18,7 +18,6 @@ package serviceimport ("context"- "encoding/json""errors""fmt""runtime"@@ -64,10 +63,12 @@ type Ingestor struct {dispatchCancel context.CancelFunc// Configuration- maxConcurrency int32- supportedDocTypes []string- version string- heartbeatInterval time.Duration+ maxConcurrency int32+ supportedDocTypes []string+ version string+ heartbeatInterval time.Duration+ memoryReconcileInterval time.Duration+ memoryReconcileBatchSize int// Runtime statecurrentTasks map[string]struct{} // set of task IDs currently claimed by a worker@@ -111,7 +112,8 @@ type Ingestor struct {kcEmbedding stringkcConcurrency int32 // number of parallel dataset-level compile workers- compileWg sync.WaitGroup+ compileWg sync.WaitGroup+ memoryReconcileWg sync.WaitGroup// runDocumentTask dispatches to the migrated task handler path.// Tests may override this to verify branch routing without invoking@@ -121,8 +123,13 @@ type Ingestor struct {// runMemoryTask dispatches one async memory-extraction task. Tests may// override this to inject a panicking/failing runner without a live DB or// real MemoryMessageService. Defaults to defaultRunMemoryTask, which calls- // memorySvc.HandleSaveToMemoryTask with the envelope task id.- runMemoryTask func(ctx context.Context, taskID string, payload map[string]any) error+ // memorySvc.HandleSaveToMemoryTask with the envelope task id and a unique+ // database lease owner.+ runMemoryTask func(ctx context.Context, taskID, leaseOwner string) (servicepkg.MemoryTaskDisposition, error)++ // reconcileMemoryTasks republishes due durable memory task wake-ups. Tests+ // may replace it to verify lifecycle behavior without a database or broker.+ reconcileMemoryTasks func(ctx context.Context) error// cancelCheck is polled periodically (every 3s) during task execution.// When it returns true the task's context is cancelled, which causes the@@ -151,25 +158,28 @@ func NewIngestor(name string, maxConcurrency int32, supportedTypes []string) *IndispatchCtx, dispatchCancel := context.WithCancel(context.Background())id := utility.GenerateUUID()ingestor := &Ingestor{- id: id,- name: name,- ctx: ctx,- cancel: cancel,- dispatchCtx: dispatchCtx,- dispatchCancel: dispatchCancel,- maxConcurrency: maxConcurrency,- supportedDocTypes: supportedTypes,- version: "1.0.0",- currentTasks: make(map[string]struct{}),- activeLeases: make(map[*Heartbeat]*activeLease),- workerQueue: make(chan *worker, maxConcurrency),- ShutdownCh: make(chan struct{}, 1),- ingestionTaskSvc: servicepkg.NewIngestionTaskService(),- docState: newDocStateUpdater(),- heartbeatInterval: defaultHeartbeatInterval,+ id: id,+ name: name,+ ctx: ctx,+ cancel: cancel,+ dispatchCtx: dispatchCtx,+ dispatchCancel: dispatchCancel,+ maxConcurrency: maxConcurrency,+ supportedDocTypes: supportedTypes,+ version: "1.0.0",+ currentTasks: make(map[string]struct{}),+ activeLeases: make(map[*Heartbeat]*activeLease),+ workerQueue: make(chan *worker, maxConcurrency),+ ShutdownCh: make(chan struct{}, 1),+ ingestionTaskSvc: servicepkg.NewIngestionTaskService(),+ docState: newDocStateUpdater(),+ heartbeatInterval: defaultHeartbeatInterval,+ memoryReconcileInterval: defaultMemoryTaskReconcileInterval,+ memoryReconcileBatchSize: defaultMemoryTaskReconcileBatchSize,}ingestor.runDocumentTask = ingestor.defaultRunDocumentTaskingestor.runMemoryTask = ingestor.defaultRunMemoryTask+ ingestor.reconcileMemoryTasks = ingestor.defaultReconcileMemoryTasksingestor.cancelCheck = ingestor.defaultCancelCheckingestor.checkpointExists = canvas.RedisCheckpointExistsingestor.kcConcurrency = maxConcurrency // parallel dataset-level compile workers default to the task width@@ -187,6 +197,11 @@ const consumeErrorBackoff = 1 * time.Secondconst taskPullRequestTimeout = 1 * time.Second+const (+ defaultMemoryTaskReconcileInterval = 10 * time.Second+ defaultMemoryTaskReconcileBatchSize = 100+)+func (e *Ingestor) Start() error {common.Info(fmt.Sprintf("Ingestor %s initialized", e.id))e.startOnce.Do(func() {@@ -213,11 +228,12 @@ func (e *Ingestor) start() error {}}- // Start the task worker pool and the dataset-level compile consumer as- // owned goroutines joined by Stop via workerWg/compileWg. Start follows+ // Start the task workers, memory reconciler, and dataset compile consumer as+ // owned goroutines joined by Stop. Start follows// the standard lifecycle contract: it returns immediately after kicking// these off rather than blocking on the consume loop itself.e.startWorkerPool()+ e.startMemoryTaskReconciler()e.startDatasetKnowledgeCompile()// Run the main tasks.RAGFLOW dispatcher off the caller's goroutine so Start@@ -228,6 +244,52 @@ func (e *Ingestor) start() error {return nil}+// startMemoryTaskReconciler starts the owned loop that recovers durable memory+// tasks whose initial wake-up was lost or whose retry became due.+func (e *Ingestor) startMemoryTaskReconciler() {+ if e.memorySvc == nil || dao.DB == nil || e.reconcileMemoryTasks == nil {+ return+ }+ e.memoryReconcileWg.Add(1)+ go e.memoryTaskReconcileLoop()+}++// memoryTaskReconcileLoop performs one startup pass and then polls until the+// ingestor execution context is canceled.+func (e *Ingestor) memoryTaskReconcileLoop() {+ defer e.memoryReconcileWg.Done()+ reconcile := func() {+ if err := e.reconcileMemoryTasks(e.ctx); err != nil && !errors.Is(err, context.Canceled) {+ common.Warn(fmt.Sprintf("reconcile durable memory tasks: %v", err))+ }+ }++ reconcile()+ interval := e.memoryReconcileInterval+ if interval <= 0 {+ interval = defaultMemoryTaskReconcileInterval+ }+ ticker := time.NewTicker(interval)+ defer ticker.Stop()+ for {+ select {+ case <-e.ctx.Done():+ return+ case <-ticker.C:+ reconcile()+ }+ }+}++// defaultReconcileMemoryTasks publishes one bounded batch of due wake-ups.+func (e *Ingestor) defaultReconcileMemoryTasks(ctx context.Context) error {+ batchSize := e.memoryReconcileBatchSize+ if batchSize <= 0 {+ batchSize = defaultMemoryTaskReconcileBatchSize+ }+ return e.memorySvc.ReconcileMemoryTasks(ctx, batchSize)+}+// consumeLoop is the worker dispatcher for tasks.RAGFLOW. It waits for an// available worker before synchronously pulling one task, preventing prefetch// and limiting this ingestor to one outstanding broker request.@@ -412,18 +474,7 @@ func (e *Ingestor) handleAndExecute(handle common.TaskHandle) {e.ackHandle(hb, handle, taskMessage.TaskID)return}- var payload map[string]any- if len(taskMessage.Payload) == 0 || json.Unmarshal(taskMessage.Payload, &payload) != nil {- common.Warn(fmt.Sprintf("memory task %s has no parseable payload, ack", taskMessage.TaskID))- e.ackHandle(hb, handle, taskMessage.TaskID)- return- }- taskCtx := taskpkg.NewMemoryTaskContextForScheduling(e.ctx, taskMessage.TaskID, payload, handle)- if !e.claimTask(taskMessage.TaskID) {- common.Warn(fmt.Sprintf("memory task %s redelivered while worker still processing, renew lease", taskMessage.TaskID))- e.renewDuplicateHandle(hb, handle, taskMessage.TaskID)- return- }+ taskCtx := taskpkg.NewMemoryTaskContextForScheduling(e.ctx, taskMessage.TaskID, handle)e.executeMemoryTaskWithHeartbeat(e.ctx, taskCtx, hb)return}@@ -589,30 +640,20 @@ func (e *Ingestor) executeMemoryTask(ctx context.Context, taskCtx *taskpkg.TaskCfunc (e *Ingestor) executeMemoryTaskWithHeartbeat(ctx context.Context, taskCtx *taskpkg.TaskContext, hb *Heartbeat) {taskID := taskCtx.ID()- var (- settleAck bool- settleNack bool- )+ settleAck := falsedefer func() {hb.Stop()if r := recover(); r != nil {common.Error(fmt.Sprintf("memory task %s panicked: %v", taskID, r), fmt.Errorf("%v", r))- settleNack = true}if taskCtx.Handle == nil || e.leaseAbandoned(hb) {- e.releaseTask(taskID)return}if settleAck {if err := taskCtx.Handle.Ack(); err != nil {common.Error(fmt.Sprintf("ack memory task %s", taskID), err)}- } else if settleNack {- if err := taskCtx.Handle.Nack(); err != nil {- common.Error(fmt.Sprintf("nack memory task %s", taskID), err)- }}- e.releaseTask(taskID)}()common.Info(fmt.Sprintf("Starting memory task %s", taskID))@@ -625,29 +666,24 @@ func (e *Ingestor) executeMemoryTaskWithHeartbeat(ctx context.Context, taskCtx *settleAck = truereturn}- if err := e.runMemoryTask(ctx, taskID, taskCtx.MemoryPayload); err != nil {- // defaultRunMemoryTask wraps terminal outcomes in ErrMemoryTaskTerminal- // (durable progress=-1 written, completed progress>=1.0, or no row to- // retry). Everything else is transient and must be redelivered rather- // than dropped.- if errors.Is(err, servicepkg.ErrMemoryTaskTerminal) {- common.Error(fmt.Sprintf("memory task %s failed terminally, ack", taskID), err)- settleAck = true- return- }- common.Error(fmt.Sprintf("memory task %s failed transiently, nack for redelivery", taskID), err)- settleNack = true+ leaseOwner := fmt.Sprintf("%s:%s", e.id, utility.GenerateUUID())+ disposition, err := e.runMemoryTask(ctx, taskID, leaseOwner)+ if err != nil {+ common.Error(fmt.Sprintf("memory task %s execution failed", taskID), err)+ }+ if disposition == servicepkg.MemoryTaskAcknowledge {+ settleAck = true+ common.Info(fmt.Sprintf("Memory task %s delivery acknowledged", taskID))return}- common.Info(fmt.Sprintf("Memory task %s completed", taskID))- settleAck = true+ common.Warn(fmt.Sprintf("memory task %s delivery left unsettled for durable recovery", taskID))}// defaultRunMemoryTask is the production memory-task runner. It is held behind// the runMemoryTask field so tests can substitute a panicking/failing runner// without a live DB or real MemoryMessageService.-func (e *Ingestor) defaultRunMemoryTask(ctx context.Context, taskID string, payload map[string]any) error {- return e.memorySvc.HandleSaveToMemoryTask(ctx, taskID, payload)+func (e *Ingestor) defaultRunMemoryTask(ctx context.Context, taskID, leaseOwner string) (servicepkg.MemoryTaskDisposition, error) {+ return e.memorySvc.HandleSaveToMemoryTask(ctx, taskID, leaseOwner)}func (e *Ingestor) executeTask(ctx context.Context, taskCtx *taskpkg.TaskContext) {@@ -1177,6 +1213,7 @@ func (e *Ingestor) Stop(ctx context.Context) {e.cancel()e.workerWg.Wait()e.compileWg.Wait()+ e.memoryReconcileWg.Wait()close(waitDone)}()diff --git a/internal/ingestion/service/ingestor_lifecycle_test.go b/internal/ingestion/service/ingestor_lifecycle_test.goindex 3bf1accb5c..b30c3c9c53 100644--- a/internal/ingestion/service/ingestor_lifecycle_test.go+++ b/internal/ingestion/service/ingestor_lifecycle_test.go@@ -244,16 +244,15 @@ func TestStopDeadlineLeavesStuckMemoryHandleUnsettled(t *testing.T) {release := make(chan struct{})started := make(chan struct{})- ingestor.runMemoryTask = func(context.Context, string, map[string]any) error {+ ingestor.runMemoryTask = func(context.Context, string, string) (servicepkg.MemoryTaskDisposition, error) {close(started)<-release- return nil+ return servicepkg.MemoryTaskAcknowledge, nil}handle := &fakeTaskHandle{msg: common.TaskMessage{TaskID: "memory-stop-timeout",TaskType: common.TaskTypeMemory,- Payload: []byte(`{}`),}}w := <-ingestor.workerQueuew.inbox <- handle@@ -474,6 +473,42 @@ func TestStartSchedulesCreatedTasks(t *testing.T) {}}+// TestMemoryTaskReconcilerRunsAtStartupAndStopsWithIngestor verifies the+// recovery loop is owned by the ingestor lifecycle.+func TestMemoryTaskReconcilerRunsAtStartupAndStopsWithIngestor(t *testing.T) {+ db := testutil.SetupTestDB(t)+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ ingestor := newUnitIngestor("test-memory-reconciler", 1, nil)+ ingestor.memorySvc = &servicepkg.MemoryMessageService{}+ ingestor.memoryReconcileInterval = time.Hour+ called := make(chan struct{}, 1)+ ingestor.reconcileMemoryTasks = func(context.Context) error {+ called <- struct{}{}+ return nil+ }+ ingestor.startMemoryTaskReconciler()++ select {+ case <-called:+ case <-time.After(time.Second):+ t.Fatal("memory task reconciler did not run its startup pass")+ }++ ingestor.cancel()+ stopped := make(chan struct{})+ go func() {+ ingestor.memoryReconcileWg.Wait()+ close(stopped)+ }()+ select {+ case <-stopped:+ case <-time.After(time.Second):+ t.Fatal("memory task reconciler did not stop with the ingestor context")+ }+}+// TestWorkerDispatcherDoesNotActivateTaskUntilWorkerReceivesTask prevents a busy// worker from prefetching its next task. task-2 must remain SCHEDULED while// task-1 still occupies the only worker.diff --git a/internal/ingestion/task/task_context.go b/internal/ingestion/task/task_context.goindex d41dfcb1ec..74eb4b8c74 100644--- a/internal/ingestion/task/task_context.go+++ b/internal/ingestion/task/task_context.go@@ -32,16 +32,16 @@ type TaskKind intconst (// TaskKindIngestion is an ingestion document task (IngestionTask set).TaskKindIngestion TaskKind = iota- // TaskKindMemory is an async memory-extraction task (MemoryPayload set,- // IngestionTask nil). It shares the worker pool with ingestion tasks but+ // TaskKindMemory is an async memory-extraction task (IngestionTask nil). It+ // shares the worker pool with ingestion tasks but// runs through executeMemoryTask instead of the ingestion state machine.TaskKindMemory)// TaskContext holds the execution inputs for an ingestion document task or a// memory-extraction task. Ingestion tasks populate IngestionTask and the-// document/KB/tenant chain; memory tasks populate MemoryPayload and leave-// IngestionTask nil.+// document/KB/tenant chain; memory tasks carry only the durable task id and+// leave IngestionTask nil.type TaskContext struct {Ctx context.Context@@ -57,18 +57,10 @@ type TaskContext struct {PipelineID stringFile any- // MemoryPayload carries the raw task_type="memory" message body for- // memory tasks (memory_id/source_id/message_dict). Only set for- // TaskKindMemory.- MemoryPayload map[string]any-- // taskID is the envelope task identifier (TaskMessage.TaskID) that the- // scheduler claims and later releases. It is the authoritative identity- // for claim, release, and logging across both task kinds. Unexported so- // the claim key cannot be mutated between admission (claim) and- // settlement (release), which would leak the claim and permanently block- // redelivery. There is deliberately no fallback chain in ID(): identity- // lives only here, never re-derived from IngestionTask or MemoryPayload.+ // taskID is the authoritative TaskMessage identity used for admission,+ // execution, settlement, and logging. It is unexported so it cannot change+ // while a worker owns the delivery. There is deliberately no fallback chain+ // in ID(): identity lives only here, never in task-specific data.taskID string// Handle is the message-queue ack handle for the task message that scheduled@@ -77,26 +69,21 @@ type TaskContext struct {// - TaskKindIngestion: ack on a durably-persisted terminal status and// nack otherwise (e.g. shutdown mid-task) so the message is redelivered// and resumed after restart.- // - TaskKindMemory: ack on success and on terminal failure (task absent,- // already-failed, or progress=-1 persisted by HandleSaveToMemoryTask);- // nack on transient failure (task-load DB error before any marker, or- // LLM/network error that did not reach progress=-1) so the message is- // redelivered. See executeMemoryTask.+ // - TaskKindMemory: ack when the durable task runner permits it; otherwise+ // leave the delivery unsettled for broker or reconciler recovery.Handle common.TaskHandle}// NewMemoryTaskContextForScheduling creates a lightweight TaskContext for a// memory-extraction task. taskID is the envelope TaskMessage.TaskID that the-// scheduler claims and will release; it is the authoritative identity for the-// whole memory task lifecycle. Only the scheduling-related fields are set, not-// the full ingestion business data.-func NewMemoryTaskContextForScheduling(ctx context.Context, taskID string, payload map[string]any, handle common.TaskHandle) *TaskContext {+// scheduler receives; it is the authoritative identity for the whole memory+// task lifecycle. Only scheduling fields are set, not full business data.+func NewMemoryTaskContextForScheduling(ctx context.Context, taskID string, handle common.TaskHandle) *TaskContext {return &TaskContext{- Ctx: ctx,- Kind: TaskKindMemory,- taskID: taskID,- MemoryPayload: payload,- Handle: handle,+ Ctx: ctx,+ Kind: TaskKindMemory,+ taskID: taskID,+ Handle: handle,}}@@ -113,7 +100,7 @@ func NewTaskContextForScheduling(ctx context.Context, task *entity.IngestionTask// ID returns the task identifier for claim/release and logging. It is the// envelope TaskMessage.TaskID captured at construction — there is deliberately-// no fallback to IngestionTask or MemoryPayload, so identity is never+// no fallback to IngestionTask or broker payload, so identity is never// re-derived from a source that could disagree with the claim key.func (c *TaskContext) ID() string {if c == nil {diff --git a/internal/service/memory.go b/internal/service/memory.goindex db9f9d29d5..07932b5cf9 100644--- a/internal/service/memory.go+++ b/internal/service/memory.go@@ -1676,7 +1676,7 @@ func (s *MemoryService) GetMemoryConfig(ctx context.Context, userID, memoryID stfunc (s *MemoryService) getMemoryConfig(ctx context.Context, memoryID string) (*CreateMemoryResponse, error) {memory, err := s.memoryDAO.GetWithOwnerNameByID(ctx, dao.DB, memoryID)if err != nil {- return nil, fmt.Errorf("memory '%s' not found", memoryID)+ return nil, fmt.Errorf("get memory %q: %w", memoryID, err)}return formatRetDataFromMemoryListItem(memory), nil}diff --git a/internal/service/memory_extractor.go b/internal/service/memory_extractor.goindex d6e5d8735f..4e07195848 100644--- a/internal/service/memory_extractor.go+++ b/internal/service/memory_extractor.go@@ -39,6 +39,7 @@ import ("encoding/json""errors""fmt"+ "strconv""strings""time"@@ -55,101 +56,350 @@ import (// server-local wall-clock string ("YYYY-MM-DD HH:MM:SS"), never UTC-shifted.const memoryTimeLayout = "2006-01-02 15:04:05"+const (+ memoryTaskLeaseTTL = 2 * time.Minute+ memoryTaskRetryInitialDelay = 5 * time.Second+ memoryTaskRetryMaxDelay = 5 * time.Minute+ memoryTaskFailureWriteTimeout = 5 * time.Second+ memoryTaskMaxAttempts = 10+)++var (+ memoryTaskLeaseRenewInterval = 30 * time.Second+ memoryTaskLeaseRenewTimeout = 10 * time.Second+)++var errPermanentMemoryTask = errors.New("memory: permanent task failure")++// classifyMemoryTaskDependencyError marks confirmed missing or unusable+// dependencies as permanent while preserving transient lookup failures.+func classifyMemoryTaskDependencyError(err error) error {+ if errors.Is(err, gorm.ErrRecordNotFound) || errors.Is(err, errModelConfigUnavailable) {+ return fmt.Errorf("%w: %w", errPermanentMemoryTask, err)+ }+ return err+}+// memoryNow is the wall clock behind every memory timestamp. Tests pin it// to a fixed instant in a fixed location so the server-local assertions are// deterministic on any host, including UTC CI runners.var memoryNow = time.Now-// ErrMemoryTaskTerminal marks a memory-task failure that already has a durable-// terminal outcome (task row absent, or progress already persisted as failed),-// so the caller must Ack rather than Nack/redeliver. Transient failures (DB-// read hiccup, LLM/network errors before any durable marker) return plain-// errors so executeMemoryTask can Nack and let the message be redelivered.-var ErrMemoryTaskTerminal = errors.New("memory: terminal task failure, do not redeliver")+// MemoryTaskDisposition tells the broker consumer whether a durable memory+// task delivery can be acknowledged. An unsettled delivery is recovered by+// broker redelivery or the database reconciler.+type MemoryTaskDisposition uint8-// errMemoryTaskRetryable marks a failure that must leave the broker message-// unsettled. It is used when extracted messages may already be stored but the-// durable task state could not be recorded; stable chunk ids make that retry-// safe.-var errMemoryTaskRetryable = errors.New("memory: retryable task failure")+const (+ // MemoryTaskLeaveUnsettled preserves the delivery for durable recovery.+ MemoryTaskLeaveUnsettled MemoryTaskDisposition = iota+ // MemoryTaskAcknowledge permits the consumer to settle the delivery.+ MemoryTaskAcknowledge+)// extractedMemory is one LLM-extracted memory item ready for persistence.type extractedMemory struct {- MessageType string- Content string- ValidAt string- InvalidAt string // empty means still valid+ MessageID int64 `json:"message_id"`+ MessageType string `json:"message_type"`+ Content string `json:"content"`+ ValidAt string `json:"valid_at"`+ InvalidAt string `json:"invalid_at,omitempty"` // empty means still valid}-// HandleSaveToMemoryTask processes one queued memory task. Mirrors-// Python handle_save_to_memory_task: validate the task row, then-// extract + persist, settling task progress on the way out.-//-// taskID is the authoritative identity passed explicitly by the Ingestor (the-// envelope TaskID); it is never re-derived from the payload. The payload-// carries only business parameters (memory_id / source_id / message_dict).-//-// The returned error is wrapped in ErrMemoryTaskTerminal when the failure has-// already produced a durable terminal outcome (dependency/config error, task-// row absent, task already failed, or extraction failed after progress=-1 was-// persisted). Transient failures (a task-load DB error before any marker was-// written) return an unwrapped error so the caller can Nack and redeliver.-func (s *MemoryMessageService) HandleSaveToMemoryTask(ctx context.Context, taskID string, payload map[string]any) error {- if s == nil || s.taskDAO == nil || s.memories == nil {- return fmt.Errorf("%w: memory: nil MemoryMessageService or memory dependency", ErrMemoryTaskTerminal)- }- memoryID, _ := payload["memory_id"].(string)- sourceID := payloadInt64(payload["source_id"])- msgDict, _ := payload["message_dict"].(map[string]any)- msg := MemoryMessage{- UserID: payloadString(msgDict["user_id"]),- AgentID: payloadString(msgDict["agent_id"]),- SessionID: payloadString(msgDict["session_id"]),- UserInput: payloadString(msgDict["user_input"]),- AgentResponse: payloadString(msgDict["agent_response"]),+type memoryExtractionCheckpoint struct {+ MessageID string `json:"message_id"`+ MessageType string `json:"message_type"`+ Content string `json:"content"`+ ValidAt string `json:"valid_at"`+ InvalidAt string `json:"invalid_at,omitempty"`+}++// HandleSaveToMemoryTask claims and resumes one durable memory task. The NATS+// delivery is only a wake-up; task input and checkpoints always come from the+// memory_task row identified by taskID.+func (s *MemoryMessageService) HandleSaveToMemoryTask(ctx context.Context, taskID, leaseOwner string) (MemoryTaskDisposition, error) {+ if s == nil {+ return MemoryTaskAcknowledge, errors.New("memory: nil MemoryMessageService")+ }+ if taskID == "" || leaseOwner == "" {+ return MemoryTaskLeaveUnsettled, errors.New("memory: task id and lease owner are required")+ }+ if s.memoryTaskDAO == nil {+ s.memoryTaskDAO = dao.NewMemoryTaskDAO()+ }++ task, acquired, err := s.memoryTaskDAO.Claim(ctx, dao.DB, taskID, leaseOwner, memoryNow(), memoryTaskLeaseTTL)+ if err != nil {+ if errors.Is(err, gorm.ErrRecordNotFound) {+ return s.failTaskMissingDurableState(ctx, taskID)+ }+ return MemoryTaskLeaveUnsettled, fmt.Errorf("memory: claim task %s: %w", taskID, err)+ }+ if !acquired {+ switch task.State {+ case entity.MemoryTaskStateCompleted, entity.MemoryTaskStateFailed:+ return MemoryTaskAcknowledge, nil+ case entity.MemoryTaskStatePending, entity.MemoryTaskStateExtracted, entity.MemoryTaskStateStored:+ return MemoryTaskLeaveUnsettled, nil+ default:+ return MemoryTaskAcknowledge, fmt.Errorf("memory: task %s has unknown state %q", taskID, task.State)+ }+ }++ return s.runClaimedMemoryTask(ctx, task, leaseOwner)+}++// runClaimedMemoryTask renews the DB lease while the state machine advances.+func (s *MemoryMessageService) runClaimedMemoryTask(ctx context.Context, task *entity.MemoryTask, leaseOwner string) (MemoryTaskDisposition, error) {+ if task.LeaseExpiresAt == nil {+ return MemoryTaskLeaveUnsettled, fmt.Errorf("memory: claimed task %s has no lease expiration", task.TaskID)+ }+ leaseExpiresAt := *task.LeaseExpiresAt+ runCtx, cancel := context.WithCancel(ctx)+ renewErr := make(chan error, 1)+ renewDone := make(chan struct{})+ go func() {+ defer close(renewDone)+ s.renewMemoryTaskLease(runCtx, cancel, task.TaskID, leaseOwner, leaseExpiresAt, renewErr)+ }()+ defer func() {+ cancel()+ <-renewDone+ }()++ resumeTask := s.resumeMemoryTask+ if s.resumeTask != nil {+ resumeTask = s.resumeTask+ }+ err := func() (err error) {+ defer func() {+ if recovered := recover(); recovered != nil {+ err = fmt.Errorf("memory: task %s panicked: %v", task.TaskID, recovered)+ }+ }()+ return resumeTask(runCtx, task, leaseOwner)+ }()+ if err == nil {+ return MemoryTaskAcknowledge, nil+ }+ select {+ case leaseErr := <-renewErr:+ if leaseErr != nil {+ err = leaseErr+ }+ default:+ }+ return s.persistMemoryTaskFailure(ctx, task, leaseOwner, err)+}++// persistMemoryTaskFailure records the recovery decision before allowing the+// broker delivery to be acknowledged. If the write cannot be proven, the+// delivery remains unsettled and the expired lease is recovered later.+func (s *MemoryMessageService) persistMemoryTaskFailure(ctx context.Context, task *entity.MemoryTask, leaseOwner string, runErr error) (MemoryTaskDisposition, error) {+ persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), memoryTaskFailureWriteTimeout)+ defer cancel()++ var (+ updated bool+ err error+ action string+ )+ now := memoryNow()+ reportedErr := runErr+ if !errors.Is(runErr, errPermanentMemoryTask) && task.AttemptCount >= memoryTaskMaxAttempts {+ reportedErr = fmt.Errorf("memory: task failed after %d attempts: %w", task.AttemptCount, runErr)+ }+ if errors.Is(runErr, errPermanentMemoryTask) || task.AttemptCount >= memoryTaskMaxAttempts {+ action = "mark failed"+ updated, err = s.memoryTaskDAO.MarkFailed(persistCtx, dao.DB, task.TaskID, leaseOwner, reportedErr.Error(), now)+ } else {+ action = "schedule retry"+ nextRetryAt := now.Add(memoryTaskRetryDelay(task.AttemptCount))+ updated, err = s.memoryTaskDAO.ScheduleRetry(persistCtx, dao.DB, task.TaskID, leaseOwner, now, nextRetryAt, reportedErr.Error())+ }+ if err != nil {+ return MemoryTaskLeaveUnsettled, errors.Join(reportedErr, fmt.Errorf("memory: %s for task %s: %w", action, task.TaskID, err))+ }+ if !updated {+ return MemoryTaskLeaveUnsettled, errors.Join(reportedErr, fmt.Errorf("memory: %s for task %s: lease is no longer owned by this worker", action, task.TaskID))}+ return MemoryTaskAcknowledge, reportedErr+}+// failTaskMissingDurableState makes an unfinished task visible as failed+// instead of silently acknowledging a delivery that cannot resume.+func (s *MemoryMessageService) failTaskMissingDurableState(ctx context.Context, taskID string) (MemoryTaskDisposition, error) {+ if s.taskDAO == nil {+ s.taskDAO = dao.NewTaskDAO()+ }task, err := s.taskDAO.GetByID(ctx, dao.DB, taskID)if err != nil {- // Record-not-found is terminal: no row to retry against. Any other- // task-load error is transient and must be redelivered (no progress=-1- // marker was written).if errors.Is(err, gorm.ErrRecordNotFound) {- return fmt.Errorf("%w: memory: task %s is not found", ErrMemoryTaskTerminal, taskID)+ return MemoryTaskAcknowledge, fmt.Errorf("memory: task %s is not found", taskID)}- return fmt.Errorf("memory: load task %s: %w", taskID, err)+ return MemoryTaskLeaveUnsettled, fmt.Errorf("memory: load task %s after durable state lookup: %w", taskID, err)}- if task.Progress == -1 {- return fmt.Errorf("%w: memory: task %s is already failed", ErrMemoryTaskTerminal, taskID)+ if task.TaskType != common.TaskTypeMemory || task.Progress < 0 || task.Progress >= 1 {+ return MemoryTaskAcknowledge, fmt.Errorf("memory: task %s has no active durable execution state", taskID)+ }++ failure := fmt.Errorf("memory: task %s cannot resume because it is missing durable execution state after the server upgrade; save the message again", taskID)+ persistCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), memoryTaskFailureWriteTimeout)+ defer cancel()+ updated, updateErr := s.taskDAO.MarkActiveMemoryTaskFailed(persistCtx, dao.DB, taskID, failure.Error())+ if updateErr != nil {+ return MemoryTaskLeaveUnsettled, errors.Join(failure, fmt.Errorf("memory: mark task failed: %w", updateErr))+ }+ if !updated {+ return MemoryTaskAcknowledge, fmt.Errorf("%w: task is no longer active", failure)+ }+ common.Warn(failure.Error())+ return MemoryTaskAcknowledge, failure+}++// memoryTaskRetryDelay applies bounded exponential backoff using the attempt+// count incremented when the task lease was claimed.+func memoryTaskRetryDelay(attemptCount int) time.Duration {+ delay := memoryTaskRetryInitialDelay+ for attempt := 1; attempt < attemptCount && delay < memoryTaskRetryMaxDelay; attempt++ {+ delay *= 2+ if delay >= memoryTaskRetryMaxDelay {+ return memoryTaskRetryMaxDelay+ }}- // A task already extracted to completion (progress>=1.0) is a durable- // terminal outcome: a redelivery after restart (or a duplicate copy from- // another consumer) must not re-run the LLM extraction, which would insert- // duplicate memory entries. Short-circuit to success so the Ingestor Acks- // the message instead of re-executing the task.- if task.Progress >= 1.0 {+ return delay+}++// renewMemoryTaskLease retries transient renewal failures while the last+// confirmed lease has time for another attempt, and cancels execution when+// ownership is lost or the lease is too close to expiry.+func (s *MemoryMessageService) renewMemoryTaskLease(ctx context.Context, cancel context.CancelFunc, taskID, leaseOwner string, leaseExpiresAt time.Time, errCh chan<- error) {+ ticker := time.NewTicker(memoryTaskLeaseRenewInterval)+ defer ticker.Stop()+ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ renewedAt := memoryNow()+ renewCtx, renewCancel := context.WithTimeout(ctx, memoryTaskLeaseRenewTimeout)+ renewed, err := s.memoryTaskDAO.RenewLease(renewCtx, dao.DB, taskID, leaseOwner, renewedAt, memoryTaskLeaseTTL)+ renewCancel()+ if err == nil && renewed {+ leaseExpiresAt = renewedAt.Add(memoryTaskLeaseTTL)+ continue+ }+ if err != nil && memoryNow().Add(memoryTaskLeaseRenewInterval).Before(leaseExpiresAt) {+ common.Warn(fmt.Sprintf("memory: renew task %s lease failed, will retry", taskID), zap.Error(err))+ continue+ }+ if err == nil {+ err = errors.New("lease is no longer owned by this worker")+ }+ select {+ case errCh <- fmt.Errorf("memory: renew task %s lease: %w", taskID, err):+ default:+ }+ cancel()+ return+ }+ }+}++// resumeMemoryTask advances from the last durable checkpoint through+// completion without consulting the generic task progress projection.+func (s *MemoryMessageService) resumeMemoryTask(ctx context.Context, task *entity.MemoryTask, leaseOwner string) error {+ var (+ msg MemoryMessage+ inputLoaded bool+ )+ loadInput := func() error {+ if inputLoaded {+ return nil+ }+ var err error+ msg, err = memoryMessageFromTaskInput(task.Input)+ if err != nil {+ return fmt.Errorf("%w: decode task %s input: %v", errPermanentMemoryTask, task.TaskID, err)+ }+ inputLoaded = truereturn nil}- if err = s.saveExtractedToMemory(ctx, memoryID, msg, sourceID, taskID); err != nil {- if errors.Is(err, errMemoryTaskRetryable) {+ for {+ if err := ctx.Err(); err != nil {return err}- if progressErr := s.updateTaskProgress(ctx, taskID, -1, err.Error()); progressErr != nil {- return fmt.Errorf("%w: memory: mark task %s failed: %v", errMemoryTaskRetryable, taskID, progressErr)+ switch task.State {+ case entity.MemoryTaskStatePending:+ if err := loadInput(); err != nil {+ return err+ }+ extraction, extractErr := s.extractMemoryTask(ctx, task, msg)+ if extractErr != nil {+ return extractErr+ }+ encoded, encodeErr := encodeMemoryExtraction(extraction)+ if encodeErr != nil {+ return fmt.Errorf("memory: encode task %s extraction: %w", task.TaskID, encodeErr)+ }+ updated, persistErr := s.memoryTaskDAO.PersistExtraction(ctx, dao.DB, task.TaskID, leaseOwner, memoryNow(), encoded)+ if persistErr != nil {+ return fmt.Errorf("memory: persist task %s extraction: %w", task.TaskID, persistErr)+ }+ if !updated {+ return fmt.Errorf("memory: task %s lost its lease before extraction checkpoint", task.TaskID)+ }+ task.Extraction = encoded+ task.State = entity.MemoryTaskStateExtracted+ _ = s.updateTaskProgress(ctx, task.TaskID, 0.5, fmt.Sprintf("Extracted %d messages from raw dialogue.", len(extraction)))++ case entity.MemoryTaskStateExtracted:+ if err := loadInput(); err != nil {+ return err+ }+ if err := s.storeMemoryTaskExtraction(ctx, task, msg); err != nil {+ return err+ }+ updated, storeErr := s.memoryTaskDAO.MarkStored(ctx, dao.DB, task.TaskID, leaseOwner, memoryNow())+ if storeErr != nil {+ return fmt.Errorf("memory: mark task %s stored: %w", task.TaskID, storeErr)+ }+ if !updated {+ return fmt.Errorf("memory: task %s lost its lease before storage checkpoint", task.TaskID)+ }+ task.State = entity.MemoryTaskStateStored++ case entity.MemoryTaskStateStored:+ progressMsg := "Message saved successfully."+ if len(task.Extraction) == 0 {+ progressMsg = "No memory extracted from raw message."+ }+ completed, completeErr := s.memoryTaskDAO.Complete(ctx, dao.DB, task.TaskID, leaseOwner, progressMsg, memoryNow())+ if completeErr != nil {+ return fmt.Errorf("memory: complete task %s: %w", task.TaskID, completeErr)+ }+ if !completed {+ return fmt.Errorf("memory: task %s lost its lease before completion", task.TaskID)+ }+ return nil++ case entity.MemoryTaskStateCompleted, entity.MemoryTaskStateFailed:+ return nil+ default:+ return fmt.Errorf("%w: task %s has unknown state %q", errPermanentMemoryTask, task.TaskID, task.State)}- return fmt.Errorf("%w: %w", ErrMemoryTaskTerminal, err)}- return nil}-// saveExtractedToMemory mirrors Python save_extracted_to_memory_only:-// skip raw-only memories, run LLM extraction, embed and persist the-// extracted messages under the raw message's id.-func (s *MemoryMessageService) saveExtractedToMemory(ctx context.Context, memoryID string, msg MemoryMessage, sourceID int64, taskID string) error {- mem, err := s.memories.getMemoryConfig(ctx, memoryID)+// extractMemoryTask executes the LLM stage and materializes retry-stable output.+func (s *MemoryMessageService) extractMemoryTask(ctx context.Context, task *entity.MemoryTask, msg MemoryMessage) ([]extractedMemory, error) {+ if s.memories == nil {+ return nil, errors.New("memory: memory service is not initialized")+ }+ mem, err := s.memories.getMemoryConfig(ctx, task.MemoryID)if err != nil {- return err+ return nil, classifyMemoryTaskDependencyError(err)}memoryTypes := mem.MemoryTypeif len(memoryTypes) == 0 {@@ -157,35 +407,39 @@ func (s *MemoryMessageService) saveExtractedToMemory(ctx context.Context, memory}extractTypes := getTypesToExtract(memoryTypes)if len(extractTypes) == 0 {- if err := s.updateTaskProgress(ctx, taskID, 1.0, fmt.Sprintf("Memory '%s' don't need to extract.", memoryID)); err != nil {- return fmt.Errorf("%w: memory: mark task %s complete: %w", errMemoryTaskRetryable, taskID, err)- }- return nil+ return []extractedMemory{}, nil}- extracted, err := s.extractByLLM(ctx, mem, extractTypes, msg, taskID)+ extracted, err := s.extractByLLM(ctx, mem, extractTypes, msg, task.TaskID)if err != nil {- return err+ return nil, err+ }+ materialized := materializeMemoryExtraction(ctx, extracted, memoryNow())+ return materialized, nil+}++// storeMemoryTaskExtraction embeds and stores a previously checkpointed result.+func (s *MemoryMessageService) storeMemoryTaskExtraction(ctx context.Context, task *entity.MemoryTask, msg MemoryMessage) error {+ extracted, err := decodeMemoryExtraction(task.Extraction)+ if err != nil {+ return fmt.Errorf("%w: decode task %s extraction: %v", errPermanentMemoryTask, task.TaskID, err)}if len(extracted) == 0 {- if err := s.updateTaskProgress(ctx, taskID, 1.0, "No memory extracted from raw message."); err != nil {- return fmt.Errorf("%w: memory: mark task %s complete: %w", errMemoryTaskRetryable, taskID, err)- }return nil}- _ = s.updateTaskProgress(ctx, taskID, 0.5, fmt.Sprintf("Extracted %d messages from raw dialogue.", len(extracted)))-- // conversation_time is stamped as server-local wall clock, not UTC.- now := memoryNow()+ if s.memories == nil {+ return errors.New("memory: memory service is not initialized")+ }+ mem, err := s.memories.getMemoryConfig(ctx, task.MemoryID)+ if err != nil {+ return classifyMemoryTaskDependencyError(err)+ }messages := make([]map[string]any, 0, len(extracted))for _, item := range extracted {- messages = append(messages, buildExtractedMessage(generateRawMessageID(ctx), sourceID, memoryID, msg, item, now))+ messages = append(messages, buildExtractedMessage(task.SourceID, task.MemoryID, msg, item))}if err = s.embedAndSaveMessages(ctx, mem, messages); err != nil {- return err- }- if err := s.updateTaskProgress(ctx, taskID, 1.0, "Message saved successfully."); err != nil {- return fmt.Errorf("%w: memory: mark task %s complete: %w", errMemoryTaskRetryable, taskID, err)+ return classifyMemoryTaskDependencyError(err)}return nil}@@ -222,7 +476,7 @@ func (s *MemoryMessageService) extractByLLM(ctx context.Context, mem *CreateMemo}driver, modelName, apiConfig, _, err := NewModelProviderService().ResolveModelConfig(ctx, mem.TenantID, entity.ModelTypeChat, llmRef)if err != nil {- return nil, fmt.Errorf("resolve chat model: %w", err)+ return nil, fmt.Errorf("resolve chat model: %w", classifyMemoryTaskDependencyError(err))}chatModel := models.NewChatModel(driver, &modelName, apiConfig)@@ -235,32 +489,108 @@ func (s *MemoryMessageService) extractByLLM(ctx context.Context, mem *CreateMemoif resp == nil || resp.Answer == nil {return nil, errors.New("empty response from chat model")}- _ = s.updateTaskProgress(ctx, taskID, 0.35, "Get extracted result from LLM.")return parseMemoryExtraction(*resp.Answer, extractTypes), nil}-// buildExtractedMessage builds the persisted envelope for one extracted-// memory item. Field set matches buildRawMessage except message_type and-// source_id, which listMemoryMessages uses to aggregate extracts. Only-// logical message fields are set here; the doc engine maps them to-// storage fields (including tokenization) at insert time.-func buildExtractedMessage(messageID, sourceID int64, memoryID string, msg MemoryMessage, item extractedMemory, now time.Time) map[string]any {- validAt, ok := normalizeMemoryTime(item.ValidAt)- if !ok {- validAt = now.Format(memoryTimeLayout)- }- invalidAt, ok := normalizeMemoryTime(item.InvalidAt)- if strings.TrimSpace(item.InvalidAt) != "" && !ok {- invalidAt = now.Format(memoryTimeLayout)+// materializeMemoryExtraction assigns stable message ids and timestamp+// fallbacks before the extraction checkpoint is persisted.+func materializeMemoryExtraction(ctx context.Context, extracted []extractedMemory, now time.Time) []extractedMemory {+ materialized := make([]extractedMemory, len(extracted))+ for i, item := range extracted {+ item.MessageID = generateRawMessageID(ctx)+ item.ValidAt = formatMemoryTime(item.ValidAt, now)+ item.InvalidAt = strings.TrimSpace(item.InvalidAt)+ if item.InvalidAt != "" {+ item.InvalidAt = formatMemoryTime(item.InvalidAt, now)+ }+ materialized[i] = item+ }+ return materialized+}++// encodeMemoryExtraction converts typed extraction output to the JSON column type.+func encodeMemoryExtraction(extracted []extractedMemory) (entity.JSONSlice, error) {+ checkpoint := make([]memoryExtractionCheckpoint, len(extracted))+ for i, item := range extracted {+ checkpoint[i] = memoryExtractionCheckpoint{+ MessageID: strconv.FormatInt(item.MessageID, 10),+ MessageType: item.MessageType,+ Content: item.Content,+ ValidAt: item.ValidAt,+ InvalidAt: item.InvalidAt,+ }+ }+ data, err := json.Marshal(checkpoint)+ if err != nil {+ return nil, err}+ var encoded entity.JSONSlice+ if err = json.Unmarshal(data, &encoded); err != nil {+ return nil, err+ }+ return encoded, nil+}++// decodeMemoryExtraction restores typed extraction output from its checkpoint.+func decodeMemoryExtraction(encoded entity.JSONSlice) ([]extractedMemory, error) {+ data, err := json.Marshal(encoded)+ if err != nil {+ return nil, err+ }+ var checkpoint []memoryExtractionCheckpoint+ if err = json.Unmarshal(data, &checkpoint); err != nil {+ return nil, err+ }+ extracted := make([]extractedMemory, len(checkpoint))+ for i, item := range checkpoint {+ messageID, parseErr := strconv.ParseInt(item.MessageID, 10, 64)+ if parseErr != nil {+ return nil, fmt.Errorf("parse message id %q: %w", item.MessageID, parseErr)+ }+ extracted[i] = extractedMemory{+ MessageID: messageID,+ MessageType: item.MessageType,+ Content: item.Content,+ ValidAt: item.ValidAt,+ InvalidAt: item.InvalidAt,+ }+ }+ return extracted, nil+}++// memoryMessageFromTaskInput reads the dialogue persisted with the durable task.+func memoryMessageFromTaskInput(input entity.JSONMap) (MemoryMessage, error) {+ msg := MemoryMessage{+ UserID: stringFromJSONMap(input, "user_id"),+ AgentID: stringFromJSONMap(input, "agent_id"),+ SessionID: stringFromJSONMap(input, "session_id"),+ UserInput: stringFromJSONMap(input, "user_input"),+ AgentResponse: stringFromJSONMap(input, "agent_response"),+ }+ if msg.AgentID == "" {+ return MemoryMessage{}, errors.New("agent_id is required")+ }+ return msg, nil+}++// stringFromJSONMap returns the named string value or an empty string.+func stringFromJSONMap(values entity.JSONMap, key string) string {+ value, _ := values[key].(string)+ return value+}++// buildExtractedMessage builds the persisted envelope for one materialized+// memory item. Field set matches buildRawMessage except message_type and+// source_id, which listMemoryMessages uses to aggregate extracts.+func buildExtractedMessage(sourceID int64, memoryID string, msg MemoryMessage, item extractedMemory) map[string]any {var storedInvalidAt any- if invalidAt != "" {- storedInvalidAt = invalidAt+ if item.InvalidAt != "" {+ storedInvalidAt = item.InvalidAt}return map[string]any{- "id": fmt.Sprintf("%s_%d", memoryID, messageID),- "message_id": messageID,+ "id": fmt.Sprintf("%s_%d", memoryID, item.MessageID),+ "message_id": item.MessageID,"message_type": item.MessageType,"source_id": sourceID,"memory_id": memoryID,@@ -268,7 +598,7 @@ func buildExtractedMessage(messageID, sourceID int64, memoryID string, msg Memor"agent_id": msg.AgentID,"session_id": msg.SessionID,"content": item.Content,- "valid_at": validAt,+ "valid_at": item.ValidAt,"invalid_at": storedInvalidAt,"forget_at": nil,"status": true,@@ -343,9 +673,8 @@ func normalizeMemoryTime(value string) (string, bool) {return "", false}-// updateTaskProgress stamps and persists task progress, mirroring-// Python TaskService.update_progress call sites. Callers that establish a-// terminal task state must handle an error so the message can be retried.+// updateTaskProgress stamps and persists the UI progress projection. Durable+// execution decisions never read this value.func (s *MemoryMessageService) updateTaskProgress(ctx context.Context, taskID string, progress float64, msg string) error {if s == nil || s.taskDAO == nil {return errors.New("memory: nil task DAO")@@ -360,27 +689,3 @@ func (s *MemoryMessageService) updateTaskProgress(ctx context.Context, taskID st}return nil}--func payloadInt64(v any) int64 {- switch n := v.(type) {- case float64:- return int64(n)- case int64:- return n- case int:- return int64(n)- case json.Number:- i, _ := n.Int64()- return i- case string:- var i int64- fmt.Sscanf(n, "%d", &i)- return i- }- return 0-}--func payloadString(v any) string {- s, _ := v.(string)- return s-}diff --git a/internal/service/memory_extractor_test.go b/internal/service/memory_extractor_test.goindex fd305ffc1e..ea2c418e23 100644--- a/internal/service/memory_extractor_test.go+++ b/internal/service/memory_extractor_test.go@@ -20,13 +20,19 @@package serviceimport (+ "context"+ "errors""strings"+ "sync/atomic""testing""time"+ "ragflow/internal/common""ragflow/internal/dao""ragflow/internal/entity""ragflow/internal/ingestion/testutil"++ "gorm.io/gorm")// pinMemoryNow fixes the memory wall clock at the given instant for the@@ -38,6 +44,20 @@ func pinMemoryNow(t *testing.T, instant time.Time) {t.Cleanup(func() { memoryNow = orig })}+// pinMemoryTaskLeaseTimings shortens lease maintenance for focused lifecycle+// tests and restores the production values afterwards.+func pinMemoryTaskLeaseTimings(t *testing.T, interval, timeout time.Duration) {+ t.Helper()+ originalInterval := memoryTaskLeaseRenewInterval+ originalTimeout := memoryTaskLeaseRenewTimeout+ memoryTaskLeaseRenewInterval = interval+ memoryTaskLeaseRenewTimeout = timeout+ t.Cleanup(func() {+ memoryTaskLeaseRenewInterval = originalInterval+ memoryTaskLeaseRenewTimeout = originalTimeout+ })+}+// TestFormatMemoryTimeKeepsParsedWallClock: an offset-bearing or// Z-suffixed ISO timestamp must keep its own wall clock — never be// converted to UTC the way time.Time.UTC would.@@ -81,22 +101,34 @@ func TestBuildExtractedMessageValidAtSemantics(t *testing.T) {msg := MemoryMessage{UserID: "u1", AgentID: "a1", SessionID: "s1"}now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.Local)- withLLMTime := buildExtractedMessage(8, 42, "mem-1", msg, extractedMemory{+ materialized := materializeMemoryExtraction(t.Context(), []extractedMemory{{MessageType: "fact",Content: "likes coffee",ValidAt: "2026-08-20T10:05:00+08:00",- }, now)+ }}, now)+ withLLMTime := buildExtractedMessage(42, "mem-1", msg, materialized[0])if got := withLLMTime["valid_at"]; got != "2026-08-20 10:05:00" {t.Fatalf("valid_at = %v, want LLM wall clock %q", got, "2026-08-20 10:05:00")}- fallbackOnly := buildExtractedMessage(9, 42, "mem-1", msg, extractedMemory{+ materialized = materializeMemoryExtraction(t.Context(), []extractedMemory{{MessageType: "fact",Content: "likes coffee",- }, now)+ }}, now)+ fallbackOnly := buildExtractedMessage(42, "mem-1", msg, materialized[0])if got := fallbackOnly["valid_at"]; got != now.Format(memoryTimeLayout) {t.Fatalf("valid_at = %v, want fallback %q", got, now.Format(memoryTimeLayout))}++ materialized = materializeMemoryExtraction(t.Context(), []extractedMemory{{+ MessageType: "fact",+ Content: "likes coffee",+ InvalidAt: " ",+ }}, now)+ blankInvalidAt := buildExtractedMessage(42, "mem-1", msg, materialized[0])+ if got := blankInvalidAt["invalid_at"]; got != nil {+ t.Fatalf("invalid_at = %#v, want nil for whitespace-only input", got)+ }}// TestBuildExtractedMessageUsesStandardDocumentID ensures extracted items@@ -105,19 +137,822 @@ func TestBuildExtractedMessageUsesStandardDocumentID(t *testing.T) {msg := MemoryMessage{UserID: "u1", AgentID: "a1", SessionID: "s1"}now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.Local)item := extractedMemory{+ MessageID: 8,MessageType: "fact",Content: "likes coffee",+ ValidAt: now.Format(memoryTimeLayout),}- extracted := buildExtractedMessage(8, 42, "mem-1", msg, item, now)+ extracted := buildExtractedMessage(42, "mem-1", msg, item)gotID, ok := extracted["id"].(string)if !ok || gotID != "mem-1_8" {t.Fatalf("extracted message id = %#v, want %q", extracted["id"], "mem-1_8")}}-// TestUpdateTaskProgressReturnsPersistenceError ensures callers can keep the-// broker message retryable when they cannot persist the terminal task state.+// TestMemoryExtractionCheckpointRoundTripPreservesMaterializedFields verifies+// retries reuse the same message id and timestamps.+func TestMemoryExtractionCheckpointRoundTripPreservesMaterializedFields(t *testing.T) {+ want := []extractedMemory{{+ MessageID: 1700000000000000123,+ MessageType: "fact",+ Content: "likes coffee",+ ValidAt: "2026-08-20 10:05:00",+ }}+ encoded, err := encodeMemoryExtraction(want)+ if err != nil {+ t.Fatalf("encodeMemoryExtraction: %v", err)+ }+ got, err := decodeMemoryExtraction(encoded)+ if err != nil {+ t.Fatalf("decodeMemoryExtraction: %v", err)+ }+ if len(got) != 1 || got[0] != want[0] {+ t.Fatalf("checkpoint round trip = %+v, want %+v", got, want)+ }+}++// TestExtractMemoryTaskMarksMissingMemoryPermanent verifies a deleted memory+// terminates its durable task instead of entering the retry schedule forever.+func TestExtractMemoryTaskMarksMissingMemoryPermanent(t *testing.T) {+ db := testutil.SetupTestDB(t, &entity.Memory{}, &entity.User{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ svc := NewMemoryMessageService(NewMemoryService())+ _, err := svc.extractMemoryTask(t.Context(), &entity.MemoryTask{MemoryID: "missing-memory"}, MemoryMessage{})+ if !errors.Is(err, errPermanentMemoryTask) || !errors.Is(err, gorm.ErrRecordNotFound) {+ t.Fatalf("extractMemoryTask error = %v, want permanent missing-memory error", err)+ }+}++// TestExtractMemoryTaskKeepsLookupFailureRetryable verifies an unexpected+// database failure is not mistaken for a deleted dependency.+func TestExtractMemoryTaskKeepsLookupFailureRetryable(t *testing.T) {+ db := testutil.SetupTestDB(t, &entity.Task{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ svc := NewMemoryMessageService(NewMemoryService())+ _, err := svc.extractMemoryTask(t.Context(), &entity.MemoryTask{MemoryID: "memory-1"}, MemoryMessage{})+ if err == nil || errors.Is(err, errPermanentMemoryTask) {+ t.Fatalf("extractMemoryTask error = %v, want retryable database error", err)+ }+}++// TestExtractByLLMMarksMissingModelPermanent verifies a deleted model+// reference terminates its durable task before any model call is attempted.+func TestExtractByLLMMarksMissingModelPermanent(t *testing.T) {+ db := testutil.SetupTestDB(t, &entity.TenantModel{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ svc := NewMemoryMessageService(NewMemoryService())+ mem := &CreateMemoryResponse{Memory: entity.Memory{+ TenantID: "tenant-1",+ LLMID: "deleted-model-id",+ }}+ _, err := svc.extractByLLM(t.Context(), mem, []string{"semantic"}, MemoryMessage{}, "task-missing-model")+ if !errors.Is(err, errPermanentMemoryTask) || !errors.Is(err, errModelConfigUnavailable) {+ t.Fatalf("extractByLLM error = %v, want permanent missing-model error", err)+ }+}++// TestExtractByLLMMarksInvalidModelConfigPermanent verifies malformed+// provider-instance configuration terminates the durable task.+func TestExtractByLLMMarksInvalidModelConfigPermanent(t *testing.T) {+ db := testutil.SetupTestDB(t,+ &entity.TenantModel{},+ &entity.TenantModelProvider{},+ &entity.TenantModelInstance{},+ )+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ if err := db.Create(&entity.TenantModelProvider{+ ID: "provider-1",+ ProviderName: "OpenAI",+ TenantID: "tenant-1",+ }).Error; err != nil {+ t.Fatalf("create model provider: %v", err)+ }+ if err := db.Create(&entity.TenantModelInstance{+ ID: "instance-1",+ InstanceName: "default",+ ProviderID: "provider-1",+ APIKey: "test-key",+ Status: "active",+ Extra: "{",+ }).Error; err != nil {+ t.Fatalf("create model instance: %v", err)+ }++ svc := NewMemoryMessageService(NewMemoryService())+ mem := &CreateMemoryResponse{Memory: entity.Memory{+ TenantID: "tenant-1",+ LLMID: "gpt-4o@default@OpenAI",+ }}+ _, err := svc.extractByLLM(t.Context(), mem, []string{"semantic"}, MemoryMessage{}, "task-invalid-model-config")+ if !errors.Is(err, errPermanentMemoryTask) || !errors.Is(err, errModelConfigUnavailable) {+ t.Fatalf("extractByLLM error = %v, want permanent invalid-model-config error", err)+ }+}++// TestHandleSaveToMemoryTaskResumesStoredCheckpoint verifies durable state,+// rather than stale UI progress, determines where execution resumes.+func TestHandleSaveToMemoryTaskResumesStoredCheckpoint(t *testing.T) {+ pinMemoryNow(t, time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC))+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ progressMsg := "stale UI state"+ if err := db.Create(&entity.Task{+ ID: "task-1",+ DocID: "memory-1",+ TaskType: "memory",+ Progress: -1,+ ProgressMsg: &progressMsg,+ }).Error; err != nil {+ t.Fatalf("create generic task: %v", err)+ }+ if err := db.Create(&entity.MemoryTask{+ TaskID: "task-1",+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{},+ State: entity.MemoryTaskStateStored,+ }).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }++ svc := NewMemoryMessageService(nil)+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-1", "worker-1")+ if err != nil || disposition != MemoryTaskAcknowledge {+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v", disposition, err)+ }+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-1")+ if err != nil {+ t.Fatalf("load memory task: %v", err)+ }+ if stored.State != entity.MemoryTaskStateCompleted {+ t.Fatalf("memory task state = %q, want completed", stored.State)+ }+ var task entity.Task+ if err = db.First(&task, "id = ?", "task-1").Error; err != nil {+ t.Fatalf("load generic task: %v", err)+ }+ if task.Progress != 1 {+ t.Fatalf("generic task progress = %v, want 1", task.Progress)+ }+}++// TestHandleSaveToMemoryTaskRetriesCompletionFromStored verifies a failed UI+// progress write leaves the durable checkpoint at stored and the next attempt+// performs only the final transaction.+func TestHandleSaveToMemoryTaskRetriesCompletionFromStored(t *testing.T) {+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)+ originalNow := memoryNow+ memoryNow = func() time.Time { return now }+ t.Cleanup(func() { memoryNow = originalNow })++ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ progressMsg := "stored"+ if err := db.Create(&entity.Task{+ ID: "task-completion-retry",+ DocID: "memory-1",+ TaskType: "memory",+ Progress: 0.5,+ ProgressMsg: &progressMsg,+ }).Error; err != nil {+ t.Fatalf("create generic task: %v", err)+ }+ if err := db.Create(&entity.MemoryTask{+ TaskID: "task-completion-retry",+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{},+ State: entity.MemoryTaskStateStored,+ }).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }+ if err := db.Exec(`+ CREATE TRIGGER fail_memory_task_completion_progress+ BEFORE UPDATE ON task+ BEGIN+ SELECT RAISE(FAIL, 'forced completion progress failure');+ END+ `).Error; err != nil {+ t.Fatalf("create update trigger: %v", err)+ }++ svc := NewMemoryMessageService(nil)+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-completion-retry", "worker-1")+ if err == nil || disposition != MemoryTaskAcknowledge {+ t.Fatalf("first attempt disposition=%v err=%v, want acknowledged scheduled retry", disposition, err)+ }+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-completion-retry")+ if err != nil {+ t.Fatalf("load stored checkpoint: %v", err)+ }+ if stored.State != entity.MemoryTaskStateStored || stored.NextRetryAt == nil {+ t.Fatalf("first attempt state/retry = %q/%v, want stored with retry", stored.State, stored.NextRetryAt)+ }+ if err = db.Exec("DROP TRIGGER fail_memory_task_completion_progress").Error; err != nil {+ t.Fatalf("drop update trigger: %v", err)+ }+ now = *stored.NextRetryAt++ disposition, err = svc.HandleSaveToMemoryTask(t.Context(), "task-completion-retry", "worker-2")+ if err != nil || disposition != MemoryTaskAcknowledge {+ t.Fatalf("retry disposition=%v err=%v, want completed acknowledgement", disposition, err)+ }+ completed, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-completion-retry")+ if err != nil {+ t.Fatalf("load completed task: %v", err)+ }+ if completed.State != entity.MemoryTaskStateCompleted {+ t.Fatalf("retry state = %q, want completed", completed.State)+ }+}++// TestHandleSaveToMemoryTaskResumesExtractedWithoutLLM verifies an extracted+// checkpoint skips extraction even when no memory/LLM dependency is available.+func TestHandleSaveToMemoryTaskResumesExtractedWithoutLLM(t *testing.T) {+ pinMemoryNow(t, time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC))+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ progressMsg := "extracted"+ if err := db.Create(&entity.Task{+ ID: "task-extracted-resume",+ DocID: "memory-1",+ TaskType: "memory",+ Progress: 0.5,+ ProgressMsg: &progressMsg,+ }).Error; err != nil {+ t.Fatalf("create generic task: %v", err)+ }+ if err := db.Create(&entity.MemoryTask{+ TaskID: "task-extracted-resume",+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{"agent_id": "agent-1"},+ State: entity.MemoryTaskStateExtracted,+ Extraction: entity.JSONSlice{},+ }).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }++ svc := NewMemoryMessageService(nil)+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-extracted-resume", "worker-1")+ if err != nil || disposition != MemoryTaskAcknowledge {+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v", disposition, err)+ }+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-extracted-resume")+ if err != nil {+ t.Fatalf("load memory task: %v", err)+ }+ if stored.State != entity.MemoryTaskStateCompleted {+ t.Fatalf("memory task state = %q, want completed", stored.State)+ }+}++// TestHandleSaveToMemoryTaskLeavesActiveLeaseUnsettled verifies an early+// redelivery is preserved while another worker still owns the durable lease.+func TestHandleSaveToMemoryTaskLeavesActiveLeaseUnsettled(t *testing.T) {+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)+ pinMemoryNow(t, now)+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ if err := db.Create(&entity.MemoryTask{+ TaskID: "task-live-lease",+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{},+ State: entity.MemoryTaskStatePending,+ }).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }+ svc := NewMemoryMessageService(nil)+ if _, acquired, err := svc.memoryTaskDAO.Claim(t.Context(), db, "task-live-lease", "worker-1", now, memoryTaskLeaseTTL); err != nil || !acquired {+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)+ }++ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-live-lease", "worker-2")+ if err != nil || disposition != MemoryTaskLeaveUnsettled {+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want unsettled live-lease delivery", disposition, err)+ }+}++// TestHandleSaveToMemoryTaskFailsTaskMissingDurableState verifies a known+// unfinished memory task is made visibly terminal instead of silently lost.+func TestHandleSaveToMemoryTaskFailsTaskMissingDurableState(t *testing.T) {+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ progressMsg := "queued"+ if err := db.Create(&entity.Task{+ ID: "task-missing-state",+ DocID: "memory-1",+ TaskType: common.TaskTypeMemory,+ Progress: 0,+ ProgressMsg: &progressMsg,+ }).Error; err != nil {+ t.Fatalf("create generic task: %v", err)+ }++ svc := NewMemoryMessageService(nil)+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-missing-state", "worker-1")+ if err == nil || disposition != MemoryTaskAcknowledge || !strings.Contains(err.Error(), "missing durable execution state") {+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want acknowledged explicit failure", disposition, err)+ }+ var task entity.Task+ if err = db.First(&task, "id = ?", "task-missing-state").Error; err != nil {+ t.Fatalf("load generic task: %v", err)+ }+ if task.Progress != -1 || task.ProgressMsg == nil || !strings.Contains(*task.ProgressMsg, "save the message again") {+ t.Fatalf("generic task progress/message = %v/%v, want visible upgrade failure", task.Progress, task.ProgressMsg)+ }+}++// TestHandleSaveToMemoryTaskAcknowledgesUnknownID verifies a delivery without+// either durable or generic task state remains distinguishable from known work+// that must be projected as failed.+func TestHandleSaveToMemoryTaskAcknowledgesUnknownID(t *testing.T) {+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ svc := NewMemoryMessageService(nil)+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "unknown-task", "worker-1")+ if err == nil || disposition != MemoryTaskAcknowledge || !strings.Contains(err.Error(), "is not found") {+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want acknowledged unknown task", disposition, err)+ }+}++// TestHandleSaveToMemoryTaskDoesNotReportUnpersistedExtraction verifies a+// failed extraction checkpoint leaves the best-effort UI projection behind it.+func TestHandleSaveToMemoryTaskDoesNotReportUnpersistedExtraction(t *testing.T) {+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)+ pinMemoryNow(t, now)+ db := testutil.SetupTestDB(t, &entity.User{}, &entity.Memory{}, &entity.Task{}, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ if err := db.Create(&entity.Memory{+ ID: "memory-raw",+ Name: "raw memory",+ TenantID: "tenant-1",+ MemoryType: dao.MemoryTypeRaw,+ StorageType: "table",+ Permissions: "me",+ ForgettingPolicy: "FIFO",+ }).Error; err != nil {+ t.Fatalf("create memory: %v", err)+ }+ progressMsg := "queued"+ if err := db.Create(&entity.Task{+ ID: "task-checkpoint-failure",+ DocID: "memory-raw",+ TaskType: common.TaskTypeMemory,+ ProgressMsg: &progressMsg,+ }).Error; err != nil {+ t.Fatalf("create generic task: %v", err)+ }+ if err := db.Create(&entity.MemoryTask{+ TaskID: "task-checkpoint-failure",+ MemoryID: "memory-raw",+ SourceID: 42,+ Input: entity.JSONMap{"agent_id": "agent-1"},+ State: entity.MemoryTaskStatePending,+ }).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }+ if err := db.Exec(`+ CREATE TRIGGER fail_memory_extraction_checkpoint+ BEFORE UPDATE ON memory_task+ WHEN NEW.state = 'extracted'+ BEGIN+ SELECT RAISE(FAIL, 'forced extraction checkpoint failure');+ END+ `).Error; err != nil {+ t.Fatalf("create checkpoint trigger: %v", err)+ }++ svc := NewMemoryMessageService(NewMemoryService())+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-checkpoint-failure", "worker-1")+ if err == nil || disposition != MemoryTaskAcknowledge {+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want acknowledged scheduled retry", disposition, err)+ }+ var task entity.Task+ if err = db.First(&task, "id = ?", "task-checkpoint-failure").Error; err != nil {+ t.Fatalf("load generic task: %v", err)+ }+ if task.Progress != 0 || task.ProgressMsg == nil || *task.ProgressMsg != "queued" {+ t.Fatalf("generic task progress/message = %v/%v, want unchanged before checkpoint", task.Progress, task.ProgressMsg)+ }+}++// TestPersistMemoryTaskFailureSchedulesRetry verifies a transient execution+// failure is durably scheduled before the current delivery is acknowledged.+func TestPersistMemoryTaskFailureSchedulesRetry(t *testing.T) {+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)+ pinMemoryNow(t, now)+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ if err := db.Create(&entity.MemoryTask{+ TaskID: "task-retry",+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{},+ State: entity.MemoryTaskStatePending,+ }).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }++ svc := NewMemoryMessageService(nil)+ claimed, acquired, err := svc.memoryTaskDAO.Claim(t.Context(), db, "task-retry", "worker-1", now, memoryTaskLeaseTTL)+ if err != nil || !acquired {+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)+ }+ disposition, err := svc.persistMemoryTaskFailure(t.Context(), claimed, "worker-1", errors.New("temporary failure"))+ if err == nil || disposition != MemoryTaskAcknowledge {+ t.Fatalf("persistMemoryTaskFailure disposition=%v err=%v, want acknowledged failure", disposition, err)+ }+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-retry")+ if err != nil {+ t.Fatalf("load memory task: %v", err)+ }+ if stored.State != entity.MemoryTaskStatePending {+ t.Fatalf("memory task state = %q, want pending", stored.State)+ }+ wantRetryAt := now.Add(memoryTaskRetryInitialDelay)+ if stored.NextRetryAt == nil || !stored.NextRetryAt.Equal(wantRetryAt) {+ t.Fatalf("next retry at = %v, want %v", stored.NextRetryAt, wantRetryAt)+ }+ if stored.LeaseOwner != "" || stored.LeaseExpiresAt != nil {+ t.Fatalf("lease = %q/%v, want released", stored.LeaseOwner, stored.LeaseExpiresAt)+ }+ if stored.AttemptCount != 1 || stored.LastError == "" {+ t.Fatalf("attempt/error = %d/%q, want one failed attempt", stored.AttemptCount, stored.LastError)+ }+}++// TestPersistMemoryTaskFailureStopsAtMaxAttempts verifies a retryable error+// becomes terminal once the claimed-attempt limit is reached.+func TestPersistMemoryTaskFailureStopsAtMaxAttempts(t *testing.T) {+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)+ pinMemoryNow(t, now)+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ progressMsg := "queued"+ if err := db.Create(&entity.Task{+ ID: "task-max-attempts",+ DocID: "memory-1",+ TaskType: common.TaskTypeMemory,+ ProgressMsg: &progressMsg,+ }).Error; err != nil {+ t.Fatalf("create generic task: %v", err)+ }+ if err := db.Create(&entity.MemoryTask{+ TaskID: "task-max-attempts",+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{},+ State: entity.MemoryTaskStatePending,+ AttemptCount: memoryTaskMaxAttempts - 1,+ }).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }++ svc := NewMemoryMessageService(nil)+ claimed, acquired, err := svc.memoryTaskDAO.Claim(t.Context(), db, "task-max-attempts", "worker-1", now, memoryTaskLeaseTTL)+ if err != nil || !acquired {+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)+ }+ disposition, err := svc.persistMemoryTaskFailure(t.Context(), claimed, "worker-1", errors.New("empty response from chat model"))+ if err == nil || disposition != MemoryTaskAcknowledge || !strings.Contains(err.Error(), "failed after 10 attempts") {+ t.Fatalf("persistMemoryTaskFailure disposition=%v err=%v, want acknowledged terminal failure", disposition, err)+ }++ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, claimed.TaskID)+ if err != nil {+ t.Fatalf("load memory task: %v", err)+ }+ if stored.State != entity.MemoryTaskStateFailed || stored.AttemptCount != memoryTaskMaxAttempts || stored.NextRetryAt != nil {+ t.Fatalf("memory task state/attempt/retry = %q/%d/%v, want failed/%d/nil", stored.State, stored.AttemptCount, stored.NextRetryAt, memoryTaskMaxAttempts)+ }+ var task entity.Task+ if err = db.First(&task, "id = ?", claimed.TaskID).Error; err != nil {+ t.Fatalf("load generic task: %v", err)+ }+ if task.Progress != -1 || task.ProgressMsg == nil || !strings.Contains(*task.ProgressMsg, "failed after 10 attempts") {+ t.Fatalf("generic task progress/message = %v/%v, want terminal retry-limit error", task.Progress, task.ProgressMsg)+ }+}++// TestPersistMemoryTaskFailureLeavesDeliveryUnsettledWhenRetryWriteFails+// verifies the broker message is preserved when no durable retry decision can+// be recorded.+func TestPersistMemoryTaskFailureLeavesDeliveryUnsettledWhenRetryWriteFails(t *testing.T) {+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)+ pinMemoryNow(t, now)+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ if err := db.Create(&entity.MemoryTask{+ TaskID: "task-retry-write-failure",+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{},+ State: entity.MemoryTaskStatePending,+ }).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }++ svc := NewMemoryMessageService(nil)+ claimed, acquired, err := svc.memoryTaskDAO.Claim(t.Context(), db, "task-retry-write-failure", "worker-1", now, memoryTaskLeaseTTL)+ if err != nil || !acquired {+ t.Fatalf("Claim acquired=%v err=%v, want acquired", acquired, err)+ }+ if err = db.Exec(`+ CREATE TRIGGER fail_memory_task_retry_update+ BEFORE UPDATE ON memory_task+ BEGIN+ SELECT RAISE(FAIL, 'forced retry update failure');+ END+ `).Error; err != nil {+ t.Fatalf("create retry trigger: %v", err)+ }++ disposition, err := svc.persistMemoryTaskFailure(t.Context(), claimed, "worker-1", errors.New("temporary failure"))+ if err == nil || disposition != MemoryTaskLeaveUnsettled {+ t.Fatalf("persistMemoryTaskFailure disposition=%v err=%v, want unsettled failure", disposition, err)+ }+}++// TestRunClaimedMemoryTaskPersistsPanic verifies a state-machine panic is+// converted into the normal durable retry path after lease renewal stops.+func TestRunClaimedMemoryTaskPersistsPanic(t *testing.T) {+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)+ pinMemoryNow(t, now)+ pinMemoryTaskLeaseTimings(t, 10*time.Millisecond, 10*time.Millisecond)+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ expiresAt := now.Add(time.Minute)+ task := &entity.MemoryTask{+ TaskID: "task-panic-cleanup",+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{},+ State: entity.MemoryTaskStatePending,+ LeaseOwner: "worker-1",+ LeaseExpiresAt: &expiresAt,+ }+ if err := db.Create(task).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }++ svc := NewMemoryMessageService(nil)+ svc.resumeTask = func(context.Context, *entity.MemoryTask, string) error {+ panic("simulated state-machine panic")+ }+ disposition, runErr := svc.runClaimedMemoryTask(t.Context(), task, "worker-1")+ if runErr == nil || disposition != MemoryTaskAcknowledge || !strings.Contains(runErr.Error(), "panicked: simulated state-machine panic") {+ t.Fatalf("runClaimedMemoryTask disposition=%v err=%v, want acknowledged persisted panic", disposition, runErr)+ }++ time.Sleep(3 * memoryTaskLeaseRenewInterval)+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, task.TaskID)+ if err != nil {+ t.Fatalf("load memory task: %v", err)+ }+ if stored.State != entity.MemoryTaskStatePending || stored.NextRetryAt == nil || stored.LeaseOwner != "" || stored.LeaseExpiresAt != nil {+ t.Fatalf("memory task after panic = %+v, want pending scheduled retry with released lease", stored)+ }+ if !strings.Contains(stored.LastError, "panicked: simulated state-machine panic") {+ t.Fatalf("last error = %q, want persisted panic", stored.LastError)+ }+}++// TestRenewMemoryTaskLeaseRetriesTransientError verifies one failed renewal+// does not discard in-flight work while the last confirmed lease is healthy.+func TestRenewMemoryTaskLeaseRetriesTransientError(t *testing.T) {+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)+ pinMemoryNow(t, now)+ pinMemoryTaskLeaseTimings(t, 5*time.Millisecond, 10*time.Millisecond)+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ expiresAt := now.Add(time.Minute)+ if err := db.Create(&entity.MemoryTask{+ TaskID: "task-renew-transient",+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{},+ State: entity.MemoryTaskStatePending,+ LeaseOwner: "worker-1",+ LeaseExpiresAt: &expiresAt,+ }).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }+ var attempts atomic.Int32+ if err := db.Callback().Update().Before("gorm:update").Register("fail_first_memory_task_lease_renewal", func(tx *gorm.DB) {+ if attempts.Add(1) == 1 {+ tx.AddError(errors.New("transient renewal failure"))+ }+ }); err != nil {+ t.Fatalf("register renewal callback: %v", err)+ }++ svc := NewMemoryMessageService(nil)+ runCtx, cancel := context.WithCancel(t.Context())+ renewErr := make(chan error, 1)+ done := make(chan struct{})+ go func() {+ defer close(done)+ svc.renewMemoryTaskLease(runCtx, cancel, "task-renew-transient", "worker-1", expiresAt, renewErr)+ }()++ wantExpiresAt := now.Add(memoryTaskLeaseTTL)+ deadline := time.NewTimer(time.Second)+ defer deadline.Stop()+ poll := time.NewTicker(time.Millisecond)+ defer poll.Stop()+ for {+ select {+ case <-deadline.C:+ cancel()+ <-done+ t.Fatal("lease was not renewed after a transient failure")+ case <-poll.C:+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-renew-transient")+ if err != nil {+ t.Fatalf("load memory task: %v", err)+ }+ if stored.LeaseExpiresAt == nil || !stored.LeaseExpiresAt.Equal(wantExpiresAt) {+ continue+ }+ if attempts.Load() < 2 {+ t.Fatalf("renewal attempts = %d, want at least 2", attempts.Load())+ }+ if runCtx.Err() != nil {+ t.Fatalf("run context canceled after recoverable renewal error: %v", runCtx.Err())+ }+ select {+ case err := <-renewErr:+ t.Fatalf("renewal reported terminal error after recovery: %v", err)+ default:+ }+ cancel()+ <-done+ return+ }+ }+}++// TestRenewMemoryTaskLeaseTimesOutNearExpiry verifies an unresponsive renewal+// cancels execution when no full renewal interval remains on the lease.+func TestRenewMemoryTaskLeaseTimesOutNearExpiry(t *testing.T) {+ now := time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC)+ pinMemoryNow(t, now)+ pinMemoryTaskLeaseTimings(t, 5*time.Millisecond, 10*time.Millisecond)+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ expiresAt := now.Add(memoryTaskLeaseRenewInterval)+ if err := db.Create(&entity.MemoryTask{+ TaskID: "task-renew-timeout",+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{},+ State: entity.MemoryTaskStatePending,+ LeaseOwner: "worker-1",+ LeaseExpiresAt: &expiresAt,+ }).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }+ if err := db.Callback().Update().Before("gorm:update").Register("block_memory_task_lease_renewal", func(tx *gorm.DB) {+ <-tx.Statement.Context.Done()+ tx.AddError(tx.Statement.Context.Err())+ }); err != nil {+ t.Fatalf("register blocking update callback: %v", err)+ }++ svc := NewMemoryMessageService(nil)+ runCtx, cancel := context.WithCancel(t.Context())+ defer cancel()+ renewErr := make(chan error, 1)+ done := make(chan struct{})+ go func() {+ defer close(done)+ svc.renewMemoryTaskLease(runCtx, cancel, "task-renew-timeout", "worker-1", expiresAt, renewErr)+ }()++ select {+ case <-done:+ case <-time.After(time.Second):+ t.Fatal("renewMemoryTaskLease did not stop after update timeout")+ }+ if runCtx.Err() == nil {+ t.Fatal("run context was not canceled after lease renewal timeout")+ }+ select {+ case err := <-renewErr:+ if err == nil || !strings.Contains(err.Error(), context.DeadlineExceeded.Error()) {+ t.Fatalf("renew error = %v, want deadline exceeded", err)+ }+ default:+ t.Fatal("renewMemoryTaskLease did not report the timeout")+ }+}++// TestHandleSaveToMemoryTaskMarksInvalidInputFailed verifies corrupt durable+// input reaches the terminal state and updates the UI progress projection.+func TestHandleSaveToMemoryTaskMarksInvalidInputFailed(t *testing.T) {+ pinMemoryNow(t, time.Date(2026, 8, 20, 10, 5, 0, 0, time.UTC))+ db := testutil.SetupTestDB(t, &entity.Task{}, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ progressMsg := "queued"+ if err := db.Create(&entity.Task{+ ID: "task-invalid",+ DocID: "memory-1",+ TaskType: "memory",+ Progress: 0,+ ProgressMsg: &progressMsg,+ }).Error; err != nil {+ t.Fatalf("create generic task: %v", err)+ }+ if err := db.Create(&entity.MemoryTask{+ TaskID: "task-invalid",+ MemoryID: "memory-1",+ SourceID: 42,+ Input: entity.JSONMap{},+ State: entity.MemoryTaskStatePending,+ }).Error; err != nil {+ t.Fatalf("create memory task: %v", err)+ }++ svc := NewMemoryMessageService(nil)+ disposition, err := svc.HandleSaveToMemoryTask(t.Context(), "task-invalid", "worker-1")+ if err == nil || disposition != MemoryTaskAcknowledge {+ t.Fatalf("HandleSaveToMemoryTask disposition=%v err=%v, want acknowledged terminal failure", disposition, err)+ }+ stored, err := svc.memoryTaskDAO.GetByID(t.Context(), db, "task-invalid")+ if err != nil {+ t.Fatalf("load memory task: %v", err)+ }+ if stored.State != entity.MemoryTaskStateFailed || stored.NextRetryAt != nil {+ t.Fatalf("memory task state/retry = %q/%v, want failed without retry", stored.State, stored.NextRetryAt)+ }+ var task entity.Task+ if err = db.First(&task, "id = ?", "task-invalid").Error; err != nil {+ t.Fatalf("load generic task: %v", err)+ }+ if task.Progress != -1 || task.ProgressMsg == nil || !strings.Contains(*task.ProgressMsg, "agent_id is required") {+ t.Fatalf("generic task progress/message = %v/%v, want terminal input error", task.Progress, task.ProgressMsg)+ }+}++// TestMemoryTaskRetryDelay verifies retries use bounded exponential backoff.+func TestMemoryTaskRetryDelay(t *testing.T) {+ for _, test := range []struct {+ attemptCount int+ want time.Duration+ }{+ {attemptCount: 0, want: 5 * time.Second},+ {attemptCount: 1, want: 5 * time.Second},+ {attemptCount: 2, want: 10 * time.Second},+ {attemptCount: 7, want: 5 * time.Minute},+ {attemptCount: 100, want: 5 * time.Minute},+ } {+ if got := memoryTaskRetryDelay(test.attemptCount); got != test.want {+ t.Errorf("memoryTaskRetryDelay(%d) = %v, want %v", test.attemptCount, got, test.want)+ }+ }+}++// TestUpdateTaskProgressReturnsPersistenceError verifies UI projection writes+// surface persistence failures without becoming execution checkpoints.func TestUpdateTaskProgressReturnsPersistenceError(t *testing.T) {db := testutil.SetupTestDB(t, &entity.IngestionTask{})cleanup := testutil.ReplaceDBForTest(t, db)diff --git a/internal/service/memory_message_service.go b/internal/service/memory_message_service.goindex f77b688a1e..b42a892c76 100644--- a/internal/service/memory_message_service.go+++ b/internal/service/memory_message_service.go@@ -47,25 +47,26 @@// 1. For each memory id: look up the Memory (via MemoryService).// 2. Generate a raw_message_id from Redis auto-increment (namespace "memory").// 3. Build the raw_message envelope (mirrors Python:344-386).-// 4. Call embed_and_save on the memory + [raw_message].-// 5. Insert a Task row in the task table for the async extractor.-// 6. Return not-found + failed lists.+// 4. Insert the UI Task and durable MemoryTask rows atomically.+// 5. Call embed_and_save on the memory + [raw_message].+// 6. Publish a task-id wake-up for the async extractor.+// 7. Return not-found + failed lists.package serviceimport ("context"- "encoding/json""errors""fmt""time""ragflow/internal/common""ragflow/internal/dao"- "ragflow/internal/engine"redisengine "ragflow/internal/engine/redis""ragflow/internal/entity"models "ragflow/internal/entity/models""ragflow/internal/utility"++ "gorm.io/gorm")// MemoryMessage is the wire shape for QueueSaveToMemoryTask. It@@ -103,8 +104,11 @@ type QueueSaveResult struct {// MemoryMessageService is the Go port of// api.db.joint_services.memory_message_service.type MemoryMessageService struct {- memories *MemoryService- taskDAO *dao.TaskDAO+ memories *MemoryService+ taskDAO *dao.TaskDAO+ memoryTaskDAO *dao.MemoryTaskDAO+ taskPublisher TaskPublisher+ resumeTask func(context.Context, *entity.MemoryTask, string) error}// NewMemoryMessageService constructs a service bound to the@@ -113,8 +117,10 @@ type MemoryMessageService struct {// `component.SetMemorySaver(...)` at boot.func NewMemoryMessageService(memories *MemoryService) *MemoryMessageService {return &MemoryMessageService{- memories: memories,- taskDAO: dao.NewTaskDAO(),+ memories: memories,+ taskDAO: dao.NewTaskDAO(),+ memoryTaskDAO: dao.NewMemoryTaskDAO(),+ taskPublisher: NewMessageQueueTaskPublisher(),}}@@ -144,7 +150,14 @@ func (s *MemoryMessageService) QueueSaveToMemoryTask(ctx context.Context, memory// (1) Look up the memory (no access control — trusted internal queue processing).mem, err := s.memories.getMemoryConfig(ctx, memoryID)if err != nil {- res.NotFound = append(res.NotFound, memoryID)+ if errors.Is(err, gorm.ErrRecordNotFound) {+ res.NotFound = append(res.NotFound, memoryID)+ } else {+ res.Failed = append(res.Failed, MemoryFailure{+ MemoryID: memoryID,+ FailMsg: err.Error(),+ })+ }continue}// (2) + (3) build the raw_message envelope. The Go port@@ -153,33 +166,67 @@ func (s *MemoryMessageService) QueueSaveToMemoryTask(ctx context.Context, memory// schema changes.rawMessageID := generateRawMessageID(ctx)rawMessage := buildRawMessage(rawMessageID, memoryID, msg)-- if err := s.embedAndSave(ctx, mem, rawMessage); err != nil {+ task, memoryTask := buildMemoryTaskRecords(rawMessageID, memoryID, msg)+ if err := s.insertMemoryTask(ctx, task, memoryTask); err != nil {res.Failed = append(res.Failed, MemoryFailure{MemoryID: memoryID,- FailMsg: err.Error(),+ FailMsg: fmt.Sprintf("task insert: %s", err.Error()),})continue}- task := buildTaskRow(rawMessageID, memoryID)- if err := s.insertTask(ctx, task); err != nil {+ if err := s.embedAndSave(ctx, mem, rawMessage); err != nil {+ failure := fmt.Errorf("store raw memory message: %w", err)+ failed, failErr := s.memoryTaskDAO.MarkUnclaimedFailed(ctx, dao.DB, task.ID, failure.Error())+ if failErr != nil {+ failure = errors.Join(failure, fmt.Errorf("mark memory task failed: %w", failErr))+ } else if !failed {+ failure = errors.Join(failure, errors.New("memory task was claimed before raw message storage completed"))+ }res.Failed = append(res.Failed, MemoryFailure{MemoryID: memoryID,- FailMsg: fmt.Sprintf("task insert: %s", err.Error()),+ FailMsg: failure.Error(),})continue}- if err = queueMemoryTask(ctx, fmt.Sprint(task["id"]), memoryID, mem.TenantID, rawMessageID, msg); err != nil {- res.Failed = append(res.Failed, MemoryFailure{- MemoryID: memoryID,- FailMsg: err.Error(),- })++ if err = publishMemoryTaskWakeup(s.taskPublisher, task.ID); err != nil {+ common.Warn(fmt.Sprintf("memory: initial task wake-up failed; reconciler will retry: %v", err))}}return res, nil}+// ReconcileMemoryTasks publishes wake-ups for due, unleased durable memory+// tasks. Publishing is idempotent because workers must claim the DB lease+// before executing any stage.+func (s *MemoryMessageService) ReconcileMemoryTasks(ctx context.Context, limit int) error {+ if s == nil {+ return errors.New("memory: nil MemoryMessageService")+ }+ if s.memoryTaskDAO == nil {+ s.memoryTaskDAO = dao.NewMemoryTaskDAO()+ }+ if s.taskPublisher == nil {+ return errors.New("memory task publisher is not initialized")+ }++ tasks, err := s.memoryTaskDAO.ListDue(ctx, dao.DB, memoryNow(), limit)+ if err != nil {+ return fmt.Errorf("memory: list due tasks: %w", err)+ }+ var reconcileErr error+ for _, task := range tasks {+ if ctxErr := ctx.Err(); ctxErr != nil {+ return errors.Join(reconcileErr, ctxErr)+ }+ if err = publishMemoryTaskWakeup(s.taskPublisher, task.TaskID); err != nil {+ reconcileErr = errors.Join(reconcileErr, err)+ }+ }+ return reconcileErr+}+// generateRawMessageID returns the Redis auto-increment id used by the Python// side (`REDIS_CONN.generate_auto_increment_id(namespace="memory")`).func generateRawMessageID(ctx context.Context) int64 {@@ -222,17 +269,37 @@ func buildRawMessage(}}-// buildTaskRow constructs the Task row the async extractor polls.-func buildTaskRow(rawMessageID int64, memoryID string) map[string]any {- return map[string]any{- "id": newUUIDString(),- "doc_id": memoryID,- "task_type": "memory",- "progress": 0.0,- "progress_msg": "",- "begin_at": time.Now(),- "digest": fmt.Sprintf("%d", rawMessageID),+// buildMemoryTaskRecords constructs the UI projection and authoritative+// execution record that are inserted atomically before publishing a wake-up.+func buildMemoryTaskRecords(rawMessageID int64, memoryID string, msg MemoryMessage) (*entity.Task, *entity.MemoryTask) {+ taskID := newUUIDString()+ progressMsg := ""+ digest := fmt.Sprintf("%d", rawMessageID)+ beginAt := time.Now()+ task := &entity.Task{+ ID: taskID,+ DocID: memoryID,+ TaskType: common.TaskTypeMemory,+ Progress: 0,+ ProgressMsg: &progressMsg,+ BeginAt: &beginAt,+ Digest: &digest,}+ memoryTask := &entity.MemoryTask{+ TaskID: taskID,+ MemoryID: memoryID,+ SourceID: rawMessageID,+ Input: entity.JSONMap{+ "user_id": msg.UserID,+ "agent_id": msg.AgentID,+ "session_id": msg.SessionID,+ "user_input": msg.UserInput,+ "agent_response": msg.AgentResponse,+ },+ State: entity.MemoryTaskStatePending,+ LastError: "",+ }+ return task, memoryTask}func (s *MemoryMessageService) embedAndSave(ctx context.Context, mem *CreateMemoryResponse, rawMessage map[string]any) error {@@ -305,14 +372,15 @@ func (s *MemoryMessageService) embedAndSaveMessages(ctx context.Context, mem *Crreturn nil}-func (s *MemoryMessageService) insertTask(ctx context.Context, row map[string]any) error {+// insertMemoryTask persists both task records atomically.+func (s *MemoryMessageService) insertMemoryTask(ctx context.Context, task *entity.Task, memoryTask *entity.MemoryTask) error {if s == nil {return errors.New("nil MemoryMessageService")}- if s.taskDAO == nil {- s.taskDAO = dao.NewTaskDAO()+ if s.memoryTaskDAO == nil {+ s.memoryTaskDAO = dao.NewMemoryTaskDAO()}- return s.taskDAO.Create(ctx, dao.DB, taskFromRow(row))+ return s.memoryTaskDAO.CreateWithTask(ctx, dao.DB, task, memoryTask)}// newUUIDString is a thin wrapper so we can swap in a real UUID@@ -322,67 +390,17 @@ func newUUIDString() string {return utility.GenerateUUID()}-func taskFromRow(row map[string]any) *entity.Task {- digest := fmt.Sprint(row["digest"])- progressMsg := fmt.Sprint(row["progress_msg"])- beginAt, _ := row["begin_at"].(time.Time)- if beginAt.IsZero() {- now := time.Now()- beginAt = now- }- return &entity.Task{- ID: fmt.Sprint(row["id"]),- DocID: fmt.Sprint(row["doc_id"]),- TaskType: fmt.Sprint(row["task_type"]),- Progress: 0,- ProgressMsg: &progressMsg,- BeginAt: &beginAt,- Digest: &digest,- }-}--// queueMemoryTask publishes a memory-extraction task to NATS (tasks.RAGFLOW).-// taskID is the durable task row's id (buildTaskRow) and the sole identity on-// the envelope; the payload carries only business parameters — memory_id,-// source_id, and the dialogue to extract. Identity is deliberately NOT-// duplicated into the payload so the consumer never has two ids that could-// disagree (the envelope TaskID is authoritative end to end).-func queueMemoryTask(ctx context.Context, taskID, memoryID, tenantID string, rawMessageID int64, msg MemoryMessage) error {- message := map[string]any{- "memory_id": memoryID,- "tenant_id": tenantID,- "source_id": rawMessageID,- "message_dict": map[string]any{- "user_id": msg.UserID,- "agent_id": msg.AgentID,- "session_id": msg.SessionID,- "user_input": msg.UserInput,- "agent_response": msg.AgentResponse,- },- }- // Publish the memory-extraction task to NATS (tasks.RAGFLOW) so it is- // consumed by the Ingestor's shared consumer + worker pool, dispatched by- // TaskType=="memory" in handleAndExecute. This keeps Go out of the Python- // te.*.common Redis stream entirely, removing the cross-consumer- // contention that previously stole Python dataflow tasks.- mq := engine.GetMessageQueueEngine()- if mq == nil {- return errors.New("can't access message queue engine")- }- payload, err := json.Marshal(message)- if err != nil {- return fmt.Errorf("marshal memory task payload: %w", err)+// publishMemoryTaskWakeup publishes only the durable task identity. Workers+// load all execution input and checkpoints from memory_task.+func publishMemoryTaskWakeup(publisher TaskPublisher, taskID string) error {+ if publisher == nil {+ return errors.New("memory task publisher is not initialized")}taskMessage := common.TaskMessage{TaskID: taskID,TaskType: common.TaskTypeMemory,- Payload: payload,- }- tmPayload, err := json.Marshal(taskMessage)- if err != nil {- return fmt.Errorf("marshal memory task message: %w", err)}- if err = mq.PublishTask(common.TaskSubject, tmPayload); err != nil {+ if err := publisher.PublishTaskMessage(common.TaskSubject, taskMessage); err != nil {return fmt.Errorf("publish memory task %s: %w", taskID, err)}return nildiff --git a/internal/service/memory_message_service_test.go b/internal/service/memory_message_service_test.goindex 66bad6d966..97d54993fa 100644--- a/internal/service/memory_message_service_test.go+++ b/internal/service/memory_message_service_test.go@@ -25,6 +25,9 @@ import ("strings""testing""time"++ "ragflow/internal/entity"+ "ragflow/internal/ingestion/testutil")// TestQueueSaveToMemoryTask_NilService: a nil receiver surfaces@@ -67,6 +70,94 @@ func TestQueueSaveToMemoryTask_MissingAgentID(t *testing.T) {}}+// TestQueueSaveToMemoryTaskClassifiesMemoryLookupErrors keeps confirmed missing+// memories separate from transient database failures.+func TestQueueSaveToMemoryTaskClassifiesMemoryLookupErrors(t *testing.T) {+ t.Run("missing memory", func(t *testing.T) {+ db := testutil.SetupTestDB(t, &entity.Memory{}, &entity.User{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ res, err := NewMemoryMessageService(NewMemoryService()).QueueSaveToMemoryTask(+ t.Context(), []string{"missing-memory"}, MemoryMessage{AgentID: "agent-1"},+ )+ if err != nil {+ t.Fatalf("QueueSaveToMemoryTask: %v", err)+ }+ if len(res.NotFound) != 1 || res.NotFound[0] != "missing-memory" || len(res.Failed) != 0 {+ t.Fatalf("result = %+v, want missing memory in NotFound only", res)+ }+ })++ t.Run("database failure", func(t *testing.T) {+ db := testutil.SetupTestDB(t, &entity.Task{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ res, err := NewMemoryMessageService(NewMemoryService()).QueueSaveToMemoryTask(+ t.Context(), []string{"memory-1"}, MemoryMessage{AgentID: "agent-1"},+ )+ if err != nil {+ t.Fatalf("QueueSaveToMemoryTask: %v", err)+ }+ if len(res.NotFound) != 0 || len(res.Failed) != 1 || res.Failed[0].MemoryID != "memory-1" || res.Failed[0].FailMsg == "" {+ t.Fatalf("result = %+v, want database lookup failure in Failed only", res)+ }+ })+}++// TestQueueSaveToMemoryTaskMarksDurableTaskFailedWhenRawStorageFails verifies+// task state exists before raw storage and records a visible terminal outcome+// when that storage cannot start.+func TestQueueSaveToMemoryTaskMarksDurableTaskFailedWhenRawStorageFails(t *testing.T) {+ db := testutil.SetupTestDB(t, &entity.User{}, &entity.Memory{}, &entity.Task{}, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ if err := db.Create(&entity.Memory{+ ID: "memory-1",+ Name: "memory",+ TenantID: "tenant-1",+ MemoryType: 1,+ StorageType: "table",+ Permissions: "me",+ ForgettingPolicy: "FIFO",+ }).Error; err != nil {+ t.Fatalf("create memory: %v", err)+ }+ memorySvc := NewMemoryService()+ memorySvc.docEngine = nil+ svc := NewMemoryMessageService(memorySvc)+ publisher := &recordingTaskPublisher{}+ svc.taskPublisher = publisher++ res, err := svc.QueueSaveToMemoryTask(t.Context(), []string{"memory-1"}, MemoryMessage{AgentID: "agent-1"})+ if err != nil {+ t.Fatalf("QueueSaveToMemoryTask: %v", err)+ }+ if len(res.Failed) != 1 || !strings.Contains(res.Failed[0].FailMsg, "message store is not initialized") {+ t.Fatalf("result = %+v, want raw-storage failure", res)+ }+ if len(publisher.messages) != 0 {+ t.Fatalf("published messages = %d, want none after raw-storage failure", len(publisher.messages))+ }++ var memoryTask entity.MemoryTask+ if err = db.First(&memoryTask).Error; err != nil {+ t.Fatalf("load memory task: %v", err)+ }+ if memoryTask.State != entity.MemoryTaskStateFailed || !strings.Contains(memoryTask.LastError, "message store is not initialized") {+ t.Fatalf("memory task state/error = %q/%q, want failed raw-storage error", memoryTask.State, memoryTask.LastError)+ }+ var task entity.Task+ if err = db.First(&task, "id = ?", memoryTask.TaskID).Error; err != nil {+ t.Fatalf("load generic task: %v", err)+ }+ if task.Progress != -1 || task.ProgressMsg == nil || !strings.Contains(*task.ProgressMsg, "message store is not initialized") {+ t.Fatalf("generic task progress/message = %v/%v, want visible raw-storage failure", task.Progress, task.ProgressMsg)+ }+}+// TestBuildRawMessage_ValidAtServerLocal: valid_at is stamped as a// server-local wall-clock string, not UTC — otherwise memories asked at// 10:05 local show up as 02:05. The clock is pinned to a fixed instant in a@@ -137,37 +228,34 @@ func TestBuildRawMessage_EnvelopeShape(t *testing.T) {}}-// TestBuildTaskRow_Shape: the task row mirrors the Python Task-// entity's memory-task shape.-func TestBuildTaskRow_Shape(t *testing.T) {- row := buildTaskRow(99, "mem-1")- if row["task_type"] != "memory" {- t.Errorf("task_type = %v, want \"memory\"", row["task_type"])+// TestBuildMemoryTaskRecordsShape verifies the UI task and durable execution+// record carry the same identity and the worker input is persisted in the DB.+func TestBuildMemoryTaskRecordsShape(t *testing.T) {+ msg := MemoryMessage{UserID: "u1", AgentID: "a1", SessionID: "s1", UserInput: "hi", AgentResponse: "hello"}+ task, memoryTask := buildMemoryTaskRecords(99, "mem-1", msg)+ if task.TaskType != "memory" {+ t.Errorf("task_type = %v, want \"memory\"", task.TaskType)}- if row["doc_id"] != "mem-1" {- t.Errorf("doc_id = %v, want \"mem-1\"", row["doc_id"])+ if task.DocID != "mem-1" {+ t.Errorf("doc_id = %v, want \"mem-1\"", task.DocID)}- if row["progress"] != 0.0 {- t.Errorf("progress = %v, want 0.0", row["progress"])+ if task.Progress != 0.0 {+ t.Errorf("progress = %v, want 0.0", task.Progress)}- if row["progress_msg"] != "" {- t.Errorf("progress_msg = %v, want empty string", row["progress_msg"])+ if task.ProgressMsg == nil || *task.ProgressMsg != "" {+ t.Errorf("progress_msg = %v, want empty string", task.ProgressMsg)}- if row["digest"] != "99" {- t.Errorf("digest = %v, want \"99\"", row["digest"])+ if task.Digest == nil || *task.Digest != "99" {+ t.Errorf("digest = %v, want \"99\"", task.Digest)}- if id, _ := row["id"].(string); len(id) != 32 {- t.Errorf("id = %q, want 32-char uuid", id)+ if len(task.ID) != 32 {+ t.Errorf("id = %q, want 32-char uuid", task.ID)}-}--func TestTaskFromRow_InitializesProgressMessage(t *testing.T) {- task := taskFromRow(buildTaskRow(99, "mem-1"))- if task.ProgressMsg == nil {- t.Fatal("ProgressMsg is nil")+ if memoryTask.TaskID != task.ID || memoryTask.MemoryID != "mem-1" || memoryTask.SourceID != 99 {+ t.Fatalf("memory task identity = %+v, want task %s/mem-1/99", memoryTask, task.ID)}- if *task.ProgressMsg != "" {- t.Fatalf("ProgressMsg = %q, want empty string", *task.ProgressMsg)+ if memoryTask.State != "pending" || memoryTask.Input["agent_response"] != "hello" {+ t.Fatalf("memory task execution data = %+v", memoryTask)}}diff --git a/internal/service/memory_message_test.go b/internal/service/memory_message_test.goindex c268e2fcda..e8b97fa861 100644--- a/internal/service/memory_message_test.go+++ b/internal/service/memory_message_test.go@@ -175,6 +175,8 @@ func setupMemoryMessageTestDB(t *testing.T) {&entity.TenantModelProvider{},&entity.TenantModelInstance{},&entity.TenantModel{},+ &entity.Task{},+ &entity.MemoryTask{},); err != nil {t.Fatalf("failed to migrate memory test tables: %v", err)}diff --git a/internal/service/memory_queue_task_test.go b/internal/service/memory_queue_task_test.goindex 37f1e86d64..313b5997e3 100644--- a/internal/service/memory_queue_task_test.go+++ b/internal/service/memory_queue_task_test.go@@ -4,11 +4,28 @@ import ("strings""testing"+ "ragflow/internal/common""ragflow/internal/engine"natsengine "ragflow/internal/engine/nats")-// queueMemoryTask is the agent-canvas memory-save publish path. When the+// TestPublishMemoryTaskWakeupContainsOnlyDurableIdentity verifies NATS carries+// no execution input that can diverge from the database record.+func TestPublishMemoryTaskWakeupContainsOnlyDurableIdentity(t *testing.T) {+ publisher := &recordingTaskPublisher{}+ if err := publishMemoryTaskWakeup(publisher, "task-1"); err != nil {+ t.Fatalf("publishMemoryTaskWakeup: %v", err)+ }+ if publisher.subject != common.TaskSubject || len(publisher.messages) != 1 {+ t.Fatalf("published subject/messages = %q/%d", publisher.subject, len(publisher.messages))+ }+ message := publisher.messages[0]+ if message.TaskID != "task-1" || message.TaskType != common.TaskTypeMemory {+ t.Fatalf("published memory wake-up = %+v", message)+ }+}++// publishMemoryTaskWakeup is the agent-canvas memory-save publish path. When the// message queue engine is a NatsEngine whose Init failed at boot, publishing// must surface a clean error (caught by the Message component's best-effort// memory save) rather than panicking and failing the whole canvas run.@@ -17,18 +34,11 @@ func TestQueueMemoryTaskUninitializedQueueReturnsError(t *testing.T) {engine.SetMessageQueueEngine(natsengine.NewNatsEngine("127.0.0.1", 1))t.Cleanup(func() { engine.SetMessageQueueEngine(previous) })- err := queueMemoryTask(- t.Context(),- "task-1",- "mem-1",- "tenant-1",- 1,- MemoryMessage{UserID: "u1", AgentID: "agent-1", SessionID: "s1", UserInput: "hi", AgentResponse: "hello"},- )+ err := publishMemoryTaskWakeup(NewMessageQueueTaskPublisher(), "task-1")if err == nil {- t.Fatal("queueMemoryTask with uninitialized MQ engine: err = nil, want publish error")+ t.Fatal("publishMemoryTaskWakeup with uninitialized MQ engine: err = nil, want publish error")}if !strings.Contains(err.Error(), "not properly initialized") {- t.Fatalf("queueMemoryTask err = %v, want engine-not-initialized error", err)+ t.Fatalf("publishMemoryTaskWakeup err = %v, want engine-not-initialized error", err)}}diff --git a/internal/service/memory_reconciler_test.go b/internal/service/memory_reconciler_test.gonew file mode 100644index 0000000000..055670ef6a--- /dev/null+++ b/internal/service/memory_reconciler_test.go@@ -0,0 +1,94 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package service++import (+ "context"+ "errors"+ "testing"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/entity"+ "ragflow/internal/ingestion/testutil"+)++// TestReconcileMemoryTasksPublishesDueWakeups verifies the database is the+// source of recovery work and the broker message carries only durable identity.+func TestReconcileMemoryTasksPublishesDueWakeups(t *testing.T) {+ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)+ pinMemoryNow(t, now)+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ future := now.Add(time.Minute)+ for _, task := range []*entity.MemoryTask{+ {TaskID: "due", MemoryID: "memory-1", SourceID: 1, Input: entity.JSONMap{}, State: entity.MemoryTaskStatePending},+ {TaskID: "future", MemoryID: "memory-1", SourceID: 2, Input: entity.JSONMap{}, State: entity.MemoryTaskStatePending, NextRetryAt: &future},+ {TaskID: "completed", MemoryID: "memory-1", SourceID: 3, Input: entity.JSONMap{}, State: entity.MemoryTaskStateCompleted},+ } {+ if err := db.Create(task).Error; err != nil {+ t.Fatalf("create memory task %s: %v", task.TaskID, err)+ }+ }++ publisher := &recordingTaskPublisher{}+ svc := NewMemoryMessageService(nil)+ svc.taskPublisher = publisher+ if err := svc.ReconcileMemoryTasks(t.Context(), 100); err != nil {+ t.Fatalf("ReconcileMemoryTasks: %v", err)+ }+ if publisher.subject != common.TaskSubject || len(publisher.messages) != 1 {+ t.Fatalf("published subject/messages = %q/%d", publisher.subject, len(publisher.messages))+ }+ message := publisher.messages[0]+ if message.TaskID != "due" || message.TaskType != common.TaskTypeMemory {+ t.Fatalf("published wake-up = %+v", message)+ }+}++// TestReconcileMemoryTasksStopsPublishingWhenCanceled verifies shutdown stops+// the current recovery batch before another wake-up is published.+func TestReconcileMemoryTasksStopsPublishingWhenCanceled(t *testing.T) {+ now := time.Date(2026, 9, 10, 12, 0, 0, 0, time.UTC)+ pinMemoryNow(t, now)+ db := testutil.SetupTestDB(t, &entity.MemoryTask{})+ cleanup := testutil.ReplaceDBForTest(t, db)+ defer cleanup()++ for _, taskID := range []string{"due-1", "due-2"} {+ if err := db.Create(&entity.MemoryTask{+ TaskID: taskID, MemoryID: "memory-1", SourceID: 1,+ Input: entity.JSONMap{}, State: entity.MemoryTaskStatePending,+ }).Error; err != nil {+ t.Fatalf("create memory task %s: %v", taskID, err)+ }+ }++ ctx, cancel := context.WithCancel(t.Context())+ publisher := &recordingTaskPublisher{beforeReturn: func(string) { cancel() }}+ svc := NewMemoryMessageService(nil)+ svc.taskPublisher = publisher+ err := svc.ReconcileMemoryTasks(ctx, 100)+ if !errors.Is(err, context.Canceled) {+ t.Fatalf("ReconcileMemoryTasks error = %v, want context canceled", err)+ }+ if len(publisher.messages) != 1 {+ t.Fatalf("published messages = %d, want 1 before cancellation", len(publisher.messages))+ }+}diff --git a/internal/service/model_service.go b/internal/service/model_service.goindex 664429d6cc..2070879112 100644--- a/internal/service/model_service.go+++ b/internal/service/model_service.go@@ -35,6 +35,31 @@ import ("gorm.io/gorm")+// errModelConfigUnavailable marks model configuration failures that cannot be+// repaired by retrying the same task.+var errModelConfigUnavailable = errors.New("model configuration unavailable")++// modelInstanceExtra contains the instance fields consumed during model+// resolution. Other provider-specific fields remain valid and are ignored.+type modelInstanceExtra struct {+ Region string `json:"region"`+ BaseURL string `json:"base_url"`+}++// decodeModelInstanceExtra reads only the endpoint fields used for model+// resolution, allowing existing rows to retain provider-specific JSON values.+func decodeModelInstanceExtra(raw string) (modelInstanceExtra, error) {+ if strings.TrimSpace(raw) == "" {+ return modelInstanceExtra{}, nil+ }++ var extra modelInstanceExtra+ if err := json.Unmarshal([]byte(raw), &extra); err != nil {+ return modelInstanceExtra{}, err+ }+ return extra, nil+}+// parseModelName parses a composite model name in format "model@instance@provider" or "model@provider"// Returns modelName, instanceName, providerName separately.//@@ -3540,59 +3565,65 @@ func (m *ModelProviderService) GetModelConfigByID(ctx context.Context, userID stmodelEntity, err := m.modelDAO.GetByID(ctx, dao.DB, modelID)if err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {- return nil, "", nil, 0, fmt.Errorf("tenant model id=%s not found", modelID)+ return nil, "", nil, 0, fmt.Errorf("%w: tenant model id=%s not found", errModelConfigUnavailable, modelID)}return nil, "", nil, 0, err}if modelEntity.Status != "active" {- return nil, "", nil, 0, fmt.Errorf("tenant model id=%s is disabled", modelID)+ return nil, "", nil, 0, fmt.Errorf("%w: tenant model id=%s is disabled", errModelConfigUnavailable, modelID)}if !entity.ModelType(modelEntity.ModelType).Has(modelType) {- return nil, "", nil, 0, fmt.Errorf("tenant model id=%s cannot be used as %s model", modelID, modelType.String())+ return nil, "", nil, 0, fmt.Errorf("%w: tenant model id=%s cannot be used as %s model", errModelConfigUnavailable, modelID, modelType.String())}providerEntity, err := m.modelProviderDAO.GetByID(ctx, dao.DB, modelEntity.ProviderID)if err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {- return nil, "", nil, 0, fmt.Errorf("provider id=%s not found for model id=%s", modelEntity.ProviderID, modelID)+ return nil, "", nil, 0, fmt.Errorf("%w: provider id=%s not found for model id=%s", errModelConfigUnavailable, modelEntity.ProviderID, modelID)}return nil, "", nil, 0, err}if providerEntity == nil {- return nil, "", nil, 0, fmt.Errorf("provider id=%s not found for model id=%s", modelEntity.ProviderID, modelID)+ return nil, "", nil, 0, fmt.Errorf("%w: provider id=%s not found for model id=%s", errModelConfigUnavailable, modelEntity.ProviderID, modelID)}allowed, accessErr := m.tenantCanReachProviderTenant(ctx, userID, providerEntity.TenantID)+if accessErr != nil {return nil, "", nil, 0, accessErr}if !allowed {- return nil, "", nil, 0, fmt.Errorf("tenant %s has no access to provider owned by tenant %s", userID, providerEntity.TenantID)+ return nil, "", nil, 0, fmt.Errorf(+ "%w: tenant %s has no access to provider owned by tenant %s",+ errModelConfigUnavailable,+ userID,+ providerEntity.TenantID,+ )}instanceEntity, err := m.modelInstanceDAO.GetByID(ctx, dao.DB, modelEntity.InstanceID)if err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {- return nil, "", nil, 0, fmt.Errorf("instance id=%s not found for model id=%s", modelEntity.InstanceID, modelID)+ return nil, "", nil, 0, fmt.Errorf("%w: instance id=%s not found for model id=%s", errModelConfigUnavailable, modelEntity.InstanceID, modelID)}return nil, "", nil, 0, err}apiKey := instanceEntity.APIKey- var extra map[string]string- if err := json.Unmarshal([]byte(instanceEntity.Extra), &extra); err != nil {- return nil, "", nil, 0, err+ extra, err := decodeModelInstanceExtra(instanceEntity.Extra)+ if err != nil {+ return nil, "", nil, 0, fmt.Errorf("%w: decode model instance configuration: %v", errModelConfigUnavailable, err)}- region := extra["region"]- baseURL := extra["base_url"]+ region := extra.Region+ baseURL := extra.BaseURLproviderInfo := dao.GetModelProviderManager().FindProvider(providerEntity.ProviderName)if providerInfo == nil {- return nil, "", nil, 0, fmt.Errorf("provider %q driver not found", providerEntity.ProviderName)+ return nil, "", nil, 0, fmt.Errorf("%w: provider %q driver not found", errModelConfigUnavailable, providerEntity.ProviderName)}modelDriver, err := newModelDriverForBaseURL(providerInfo.ModelDriver, providerEntity.ProviderName, region, baseURL)if err != nil {- return nil, "", nil, 0, err+ return nil, "", nil, 0, fmt.Errorf("%w: create model driver: %v", errModelConfigUnavailable, err)}maxTokens := 0@@ -3601,7 +3632,7 @@ func (m *ModelProviderService) GetModelConfigByID(ctx context.Context, userID st}maxTokens, err = maxTokensFromTenantModelExtra(modelEntity, maxTokens)if err != nil {- return nil, "", nil, 0, err+ return nil, "", nil, 0, fmt.Errorf("%w: read model limits: %v", errModelConfigUnavailable, err)}apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: ®ion, BaseURL: &baseURL}@@ -3634,7 +3665,7 @@ func defaultModelRefs(tenant *entity.Tenant, modelType entity.ModelType) (stringfunc (m *ModelProviderService) ResolveModelConfig(ctx context.Context, tenantID string, modelType entity.ModelType, modelRef string) (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {if strings.TrimSpace(modelRef) == "" {- return nil, "", nil, 0, fmt.Errorf("model ref is required")+ return nil, "", nil, 0, fmt.Errorf("%w: model ref is required", errModelConfigUnavailable)}if _, err := m.modelDAO.GetByID(ctx, dao.DB, modelRef); err == nil {return m.GetModelConfigByID(ctx, tenantID, modelType, modelRef)@@ -4121,7 +4152,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Coif modelName == teiModel {builtinDriver := modelModule.GetBuiltinEmbeddingModel(modelName)if builtinDriver == nil {- return nil, "", nil, 0, fmt.Errorf("builtin (TEI) embedding model %q not found", modelName)+ return nil, "", nil, 0, fmt.Errorf("%w: builtin (TEI) embedding model %q not found", errModelConfigUnavailable, modelName)}apiConfig := &modelModule.APIConfig{ApiKey: nil, Region: nil, BaseURL: &teiBaseURL}return builtinDriver, modelName, apiConfig, 0, nil@@ -4133,7 +4164,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Coif teiPure == teiModel && (teiProvider == "Builtin" || teiProvider == "") {builtinDriver := modelModule.GetBuiltinEmbeddingModel(teiPure)if builtinDriver == nil {- return nil, "", nil, 0, fmt.Errorf("builtin (TEI) embedding model %q not found", teiPure)+ return nil, "", nil, 0, fmt.Errorf("%w: builtin (TEI) embedding model %q not found", errModelConfigUnavailable, teiPure)}apiConfig := &modelModule.APIConfig{ApiKey: nil, Region: nil, BaseURL: &teiBaseURL}return builtinDriver, teiPure, apiConfig, 0, nil@@ -4169,7 +4200,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Coif pureModelName != "" {builtinDriver := modelModule.GetBuiltinEmbeddingModel(pureModelName)if builtinDriver == nil {- return nil, "", nil, 0, fmt.Errorf("builtin embedding model %q not found", pureModelName)+ return nil, "", nil, 0, fmt.Errorf("%w: builtin embedding model %q not found", errModelConfigUnavailable, pureModelName)}apiKey := ""region := ""@@ -4184,33 +4215,41 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.CopureModelName, instanceName, providerName, err := parseModelName(modelName)if err != nil {- return nil, "", nil, 0, err+ return nil, "", nil, 0, fmt.Errorf("%w: %v", errModelConfigUnavailable, err)}// Direct provider lookupprovider, provErr := m.modelProviderDAO.GetByTenantIDAndProviderName(ctx, dao.DB, tenantID, providerName)if provErr != nil {+ if errors.Is(provErr, gorm.ErrRecordNotFound) {+ return nil, "", nil, 0, fmt.Errorf("%w: provider %q lookup failed: %w", errModelConfigUnavailable, providerName, provErr)+ }return nil, "", nil, 0, fmt.Errorf("provider %q lookup failed: %w", providerName, provErr)}if provider == nil {- return nil, "", nil, 0, fmt.Errorf("provider %q not found for model %q", providerName, modelName)+ return nil, "", nil, 0, fmt.Errorf("%w: provider %q not found for model %q", errModelConfigUnavailable, providerName, modelName)}// Direct instance lookupinstance, instErr := m.modelInstanceDAO.GetByProviderIDAndInstanceName(ctx, dao.DB, provider.ID, instanceName)if instErr != nil {+ if errors.Is(instErr, gorm.ErrRecordNotFound) {+ return nil, "", nil, 0, fmt.Errorf("%w: instance %q lookup failed: %w", errModelConfigUnavailable, instanceName, instErr)+ }return nil, "", nil, 0, fmt.Errorf("instance %q lookup failed: %w", instanceName, instErr)}if instance == nil {- return nil, "", nil, 0, fmt.Errorf("instance %q not found for model %q", instanceName, modelName)+ return nil, "", nil, 0, fmt.Errorf("%w: instance %q not found for model %q", errModelConfigUnavailable, instanceName, modelName)}// Decode api_key and extra fields from the instance rowapiKey := instance.APIKey- var extra map[string]string- _ = json.Unmarshal([]byte(instance.Extra), &extra)- region := extra["region"]- baseURL := extra["base_url"]+ extra, err := decodeModelInstanceExtra(instance.Extra)+ if err != nil {+ return nil, "", nil, 0, fmt.Errorf("%w: decode model instance configuration: %v", errModelConfigUnavailable, err)+ }+ region := extra.Region+ baseURL := extra.BaseURL// Direct model lookupmodelObj, modelErr := m.modelDAO.GetByProviderIDAndInstanceIDAndModelTypeAndModelName(@@ -4221,16 +4260,16 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co// Happy path: tenant enrolled this model.// INACTIVE checkif modelObj.Status == "inactive" {- return nil, "", nil, 0, fmt.Errorf("model %q is disabled", modelName)+ return nil, "", nil, 0, fmt.Errorf("%w: model %q is disabled", errModelConfigUnavailable, modelName)}providerInfo := dao.GetModelProviderManager().FindProvider(providerName)if providerInfo == nil {- return nil, "", nil, 0, fmt.Errorf("provider %q driver not found", providerName)+ return nil, "", nil, 0, fmt.Errorf("%w: provider %q driver not found", errModelConfigUnavailable, providerName)}driver, driverErr := newModelDriverForBaseURL(providerInfo.ModelDriver, providerName, region, baseURL)if driverErr != nil {- return nil, "", nil, 0, driverErr+ return nil, "", nil, 0, fmt.Errorf("%w: create model driver: %v", errModelConfigUnavailable, driverErr)}maxTokens := 0if mi, _ := dao.GetModelProviderManager().GetModelByName(providerName, pureModelName); mi != nil {@@ -4238,7 +4277,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co}maxTokens, driverErr = maxTokensFromTenantModelExtra(modelObj, maxTokens)if driverErr != nil {- return nil, "", nil, 0, driverErr+ return nil, "", nil, 0, fmt.Errorf("%w: read model limits: %v", errModelConfigUnavailable, driverErr)}apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: ®ion, BaseURL: &baseURL}return driver, modelObj.ModelName, apiConfig, maxTokens, nil@@ -4270,7 +4309,7 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co// casing). The rest of the Go codebase uses these helpers too.targetProvider := dao.GetModelProviderManager().FindProvider(targetFactoryName)if targetProvider == nil {- return nil, "", nil, 0, fmt.Errorf("model provider config not found: %s", providerName)+ return nil, "", nil, 0, fmt.Errorf("%w: model provider config not found: %s", errModelConfigUnavailable, providerName)}var llmInfo *modelModule.Modelfor i := range targetProvider.Models {@@ -4280,11 +4319,11 @@ func (m *ModelProviderService) GetModelConfigFromProviderInstance(ctx context.Co}}if llmInfo == nil {- return nil, "", nil, 0, fmt.Errorf("model config not found: %s", modelName)+ return nil, "", nil, 0, fmt.Errorf("%w: model config not found: %s", errModelConfigUnavailable, modelName)}driver, driverErr := newModelDriverForBaseURL(targetProvider.ModelDriver, providerName, region, baseURL)if driverErr != nil {- return nil, "", nil, 0, driverErr+ return nil, "", nil, 0, fmt.Errorf("%w: create model driver: %v", errModelConfigUnavailable, driverErr)}apiConfig := &modelModule.APIConfig{ApiKey: &apiKey, Region: ®ion, BaseURL: &baseURL}maxTokens := maxTokensFromModelInfo(llmInfo, modelType)diff --git a/internal/service/model_service_test.go b/internal/service/model_service_test.goindex f0959a2d6a..0e3a06bddc 100644--- a/internal/service/model_service_test.go+++ b/internal/service/model_service_test.go@@ -500,6 +500,97 @@ func TestModelProviderServiceGetModelConfigByID(t *testing.T) {}}+func TestModelProviderServiceMissingProviderAndInstancePreserveLookupError(t *testing.T) {+ db := setupModelProviderServiceTestDB(t)+ useModelProviderServiceTestDB(t, db)+ if err := db.Create(&entity.TenantModelProvider{+ ID: "provider-1",+ TenantID: "tenant-1",+ ProviderName: "OpenAI",+ }).Error; err != nil {+ t.Fatalf("create provider: %v", err)+ }++ for _, modelRef := range []string{+ "unknown@ZHIPU-AI",+ "unknown@default@OpenAI",+ } {+ t.Run(modelRef, func(t *testing.T) {+ _, _, _, _, err := NewModelProviderService().ResolveModelConfig(+ t.Context(), "tenant-1", entity.ModelTypeEmbedding, modelRef,+ )+ if !errors.Is(err, errModelConfigUnavailable) || !errors.Is(err, gorm.ErrRecordNotFound) {+ t.Fatalf("ResolveModelConfig() error = %v, want unavailable record-not-found error", err)+ }+ if !strings.Contains(err.Error(), "lookup failed: record not found") {+ t.Fatalf("ResolveModelConfig() error = %q, want lookup failure contract", err)+ }+ })+ }+}++// TestModelProviderServiceDecodesCompatibleInstanceExtra verifies both model+// resolution paths accept empty configuration and unrelated typed fields.+func TestModelProviderServiceDecodesCompatibleInstanceExtra(t *testing.T) {+ db := setupModelProviderServiceTestDB(t)+ useModelProviderServiceTestDB(t, db)+ seedModelProviderServiceScope(t, db)++ svc := NewModelProviderService()+ resolvers := []struct {+ name string+ resolve func() (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error)+ }{+ {+ name: "model ID",+ resolve: func() (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ return svc.GetModelConfigByID(t.Context(), "user-1", entity.ModelTypeChat, "model-1")+ },+ },+ {+ name: "provider instance",+ resolve: func() (modelModule.ModelDriver, string, *modelModule.APIConfig, int, error) {+ return svc.ResolveModelConfig(t.Context(), "tenant-1", entity.ModelTypeChat, "gpt-test@default@OpenAI")+ },+ },+ }+ configs := []struct {+ name string+ raw string+ wantRegion string+ wantBaseURL string+ }{+ {name: "empty", raw: ""},+ {+ name: "unrelated typed fields",+ raw: `{"region":"us-east-1","base_url":"https://models.example.com","enabled":true,"retries":3,"options":{"mode":"custom"}}`,+ wantRegion: "us-east-1",+ wantBaseURL: "https://models.example.com",+ },+ }+ for _, config := range configs {+ t.Run(config.name, func(t *testing.T) {+ if err := db.Model(&entity.TenantModelInstance{}).+ Where("id = ?", "instance-1").+ Update("extra", config.raw).Error; err != nil {+ t.Fatalf("update instance extra: %v", err)+ }++ for _, resolver := range resolvers {+ t.Run(resolver.name, func(t *testing.T) {+ driver, _, apiConfig, _, err := resolver.resolve()+ if err != nil {+ t.Fatalf("resolve model config: %v", err)+ }+ if driver == nil || apiConfig == nil || apiConfig.Region == nil || *apiConfig.Region != config.wantRegion || apiConfig.BaseURL == nil || *apiConfig.BaseURL != config.wantBaseURL {+ t.Fatalf("resolved driver/config = %v/%+v, want region %q and base URL %q", driver, apiConfig, config.wantRegion, config.wantBaseURL)+ }+ })+ }+ })+ }+}+func TestMaxTokensFromModelInfo(t *testing.T) {maxTokens := 4096maxOutput := 1024