RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(sync): preserve published blobs and retry incomplete revisions (#20627)
Lucienedeabb021e39
diff --git a/internal/dao/mysql_session.go b/internal/dao/mysql_session.gonew file mode 100644index 0000000000..e05b7faeed--- /dev/null+++ b/internal/dao/mysql_session.go@@ -0,0 +1,38 @@+package dao++import (+ "database/sql"+ "fmt"++ "gorm.io/driver/mysql"+ "gorm.io/gorm"+)++// OpenIndependentMySQLDB opens a separate pool with the active connection+// settings. Session locks must not reserve connections from the application+// pool while their owners run database operations through that same pool.+func OpenIndependentMySQLDB(db *gorm.DB) (*sql.DB, error) {+ var config *mysql.Config+ switch dialector := db.Dialector.(type) {+ case migrationAwareDialector:+ config = dialector.Config+ case *mysql.Dialector:+ config = dialector.Config+ default:+ return nil, fmt.Errorf("independent session requires a MySQL dialector")+ }+ if config == nil || config.DSN == "" {+ return nil, fmt.Errorf("independent session requires a MySQL DSN")+ }+ driverName := config.DriverName+ if driverName == "" {+ driverName = "mysql"+ }+ pool, err := sql.Open(driverName, config.DSN)+ if err != nil {+ return nil, err+ }+ pool.SetMaxOpenConns(1)+ pool.SetMaxIdleConns(0)+ return pool, nil+}diff --git a/internal/dao/mysql_session_test.go b/internal/dao/mysql_session_test.gonew file mode 100644index 0000000000..b01e3c1b1b--- /dev/null+++ b/internal/dao/mysql_session_test.go@@ -0,0 +1,42 @@+package dao++import (+ "testing"++ "github.com/DATA-DOG/go-sqlmock"+ "gorm.io/driver/mysql"+ "gorm.io/gorm"+)++func TestIndependentMySQLDBSupportsMigrationDialector(t *testing.T) {+ for _, wrapped := range []bool{false, true} {+ name := "mysql"+ if wrapped {+ name = "migration wrapper"+ }+ t.Run(name, func(t *testing.T) {+ shared, _, err := sqlmock.New()+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { shared.Close() })+ base := mysql.New(mysql.Config{DSN: "test:test@tcp(localhost:3306)/test?parseTime=True", Conn: shared, SkipInitializeWithVersion: true}).(*mysql.Dialector)+ var dialect gorm.Dialector = base+ if wrapped {+ dialect = migrationAwareDialector{Dialector: base}+ }+ db, err := gorm.Open(dialect, &gorm.Config{})+ if err != nil {+ t.Fatal(err)+ }+ independent, err := OpenIndependentMySQLDB(db)+ if err != nil {+ t.Fatal(err)+ }+ defer independent.Close()+ if independent == shared || independent.Stats().MaxOpenConnections != 1 {+ t.Fatal("session lock must have its own bounded connection pool")+ }+ })+ }+}diff --git a/internal/service/document/sync_document.go b/internal/service/document/sync_document.goindex 6c38361d49..5395688c09 100644--- a/internal/service/document/sync_document.go+++ b/internal/service/document/sync_document.go@@ -63,6 +63,14 @@ func (s *DocumentService) Upsert(ctx context.Context, input service.DocumentUpseif contentHash == "" {contentHash = contentHashHex(input.SourceDocument.Blob)}+ // Serialize publication and all dependent side effects, including the+ // unchanged-revision path. A final fingerprint CAS cannot undo stale+ // metadata written by an earlier concurrent revision.+ unlock, err := acquireSyncDocumentLock(ctx, input.DocumentID)+ if err != nil {+ return service.DocumentUpsertResult{}, err+ }+ defer unlock()// if the 'file' is existingexisting, err := s.documentDAO.GetByID(ctx, dao.DB, input.DocumentID)@@ -81,7 +89,18 @@ func (s *DocumentService) Upsert(ctx context.Context, input service.DocumentUpsecleanupLocation := truedefer func() {if cleanupLocation {- _ = storageImpl.Remove(context.WithoutCancel(ctx), kb.ID, location)+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupBatchTimeout)+ defer cancel()+ // A downstream failure can leave the staged key published. Only+ // remove it when the database confirms it is no longer referenced.+ current, lookupErr := s.documentDAO.GetByID(cleanupCtx, dao.DB, input.DocumentID)+ if lookupErr != nil && !errorsIsRecordNotFound(lookupErr) {+ return+ }+ if current != nil && current.Location != nil && *current.Location == location {+ return+ }+ _ = storageImpl.Remove(cleanupCtx, kb.ID, location)}}()if err = storageImpl.Put(ctx, kb.ID, location, input.SourceDocument.Blob); err != nil {@@ -118,12 +137,14 @@ func (s *DocumentService) insertSyncDocument(ctx context.Context, input service.parserID, parserConfig := resolveDocumentParser(ctx, kb, filename, utility.FileType(filetype), kb.ParserConfig)doc := s.newDatasetDocument(kb, tenantID, filename, "", filetype, parserID, parserConfig, input.SourceType, int64(len(input.SourceDocument.Blob)), input.SourceDocument.Blob)doc.ID = input.DocumentID+ // A fingerprint is acknowledged only after all dependent work succeeds.+ doc.ContentHash = nil// put 'file' in mysql `document`if err = s.InsertDocument(doc); err != nil {return service.DocumentUpsertResult{}, err}- if err = s.publishSyncDocument(ctx, doc, location, contentHash); err != nil {+ if err = s.publishSyncDocument(ctx, doc, location); err != nil {return service.DocumentUpsertResult{}, s.rollbackAddFileFromKBError(ctx, doc, kb.ID, err)}// link 'file' 2 'file_system'@@ -131,7 +152,7 @@ func (s *DocumentService) insertSyncDocument(ctx context.Context, input service.return service.DocumentUpsertResult{}, s.rollbackAddFileFromKBError(ctx, doc, kb.ID, err)}// write metadata and do `auto_parse` (or not)- if err = s.afterSyncDocumentUpsert(ctx, input, doc, false); err != nil {+ if err = s.afterSyncDocumentUpsert(ctx, input, doc, false, contentHash); err != nil {return service.DocumentUpsertResult{}, err}@@ -158,7 +179,7 @@ func (s *DocumentService) updateSyncDocument(ctx context.Context, input service.doc.Suffix = suffixdoc.SourceType = input.SourceType- if err := s.publishSyncDocument(ctx, doc, location, contentHash); err != nil {+ if err := s.publishSyncDocument(ctx, doc, location); err != nil {return service.DocumentUpsertResult{}, err}// update new updated file@@ -166,14 +187,17 @@ func (s *DocumentService) updateSyncDocument(ctx context.Context, input service.return service.DocumentUpsertResult{}, err}// write metadata and do `auto_parse`- if err := s.afterSyncDocumentUpsert(ctx, input, doc, true); err != nil {+ if err := s.afterSyncDocumentUpsert(ctx, input, doc, true, contentHash); err != nil {return service.DocumentUpsertResult{}, err}return service.DocumentUpsertResult{DocID: doc.ID, Action: service.DocumentActionUpdated}, nil}// publishSyncDocument publishes the staged object key after the document row exists.-func (s *DocumentService) publishSyncDocument(ctx context.Context, doc *entity.Document, location, contentHash string) error {+func (s *DocumentService) publishSyncDocument(ctx context.Context, doc *entity.Document, location string) error {+ // Invalidate the old fingerprint atomically with the location change so+ // runners and connectors cannot skip an incompletely processed revision.+ contentHash := ""if err := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{"location": location,"content_hash": contentHash,@@ -186,7 +210,7 @@ func (s *DocumentService) publishSyncDocument(ctx context.Context, doc *entity.D}// afterSyncDocumentUpsert writes metadata and optionally enqueues parsing.-func (s *DocumentService) afterSyncDocumentUpsert(ctx context.Context, input service.DocumentUpsertInput, doc *entity.Document, rerun bool) error {+func (s *DocumentService) afterSyncDocumentUpsert(ctx context.Context, input service.DocumentUpsertInput, doc *entity.Document, rerun bool, contentHash string) error {// write metadataif len(input.SourceDocument.Metadata) > 0 && s.docEngine != nil {if err := s.SetDocumentMetadata(ctx, doc.ID, input.SourceDocument.Metadata); err != nil {@@ -194,10 +218,26 @@ func (s *DocumentService) afterSyncDocumentUpsert(ctx context.Context, input ser}}// do auto_parse if enabled- if !input.AutoParse {- return nil+ if input.AutoParse {+ if err := s.StartParseDocuments(ctx, doc, &input.TaskContext.Knowledgebase, input.TaskContext.Connector.TenantID, StartParseOptions{RerunWithDelete: rerun}); err != nil {+ return err+ }+ }+ // Defend against non-sync writers replacing the document while metadata+ // or parsing was being prepared. Never acknowledge another blob's hash.+ ackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupBatchTimeout)+ defer cancel()+ result := dao.DB.WithContext(ackCtx).Model(&entity.Document{}).+ Where("id = ? AND location = ?", doc.ID, doc.Location).+ Updates(map[string]interface{}{"content_hash": contentHash})+ if result.Error != nil {+ return result.Error+ }+ if result.RowsAffected == 0 {+ return fmt.Errorf("synced document %s changed before its fingerprint was committed", doc.ID)}- return s.StartParseDocuments(ctx, doc, &input.TaskContext.Knowledgebase, input.TaskContext.Connector.TenantID, StartParseOptions{RerunWithDelete: rerun})+ doc.ContentHash = &contentHash+ return nil}// ensureSyncDocumentPostWrite retries dependent work before an unchanged document is skipped.diff --git a/internal/service/document/sync_document_lock.go b/internal/service/document/sync_document_lock.gonew file mode 100644index 0000000000..626edef6fc--- /dev/null+++ b/internal/service/document/sync_document_lock.go@@ -0,0 +1,68 @@+package document++import (+ "context"+ "database/sql"+ "fmt"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/service"+)++// acquireSyncDocumentLock serializes sync side effects across MySQL sessions,+// including for documents that have not been inserted yet. SQLite uses the+// process lock only for the default in-memory unit fixtures.+func acquireSyncDocumentLock(ctx context.Context, docID string) (func(), error) {+ key := "sync-document:" + service.Hash128(docID)+ if dao.DB.Dialector.Name() == "sqlite" {+ // SQLite is used by the in-memory service unit fixtures. Its lock key+ // must differ from the parse lock acquired by StartParseDocuments.+ return lockDocumentParse(key), nil+ }+ pool, err := dao.OpenIndependentMySQLDB(dao.DB)+ if err != nil {+ return nil, err+ }+ unlock, err := acquireMySQLSyncDocumentLock(ctx, pool, key)+ if err != nil {+ _ = pool.Close()+ return nil, err+ }+ return func() {+ // Closing the dedicated session also releases a named lock if the+ // explicit release fails; it is never returned to the application pool.+ defer pool.Close()+ unlock()+ }, nil+}++// acquireMySQLSyncDocumentLock owns a named lock on one dedicated connection+// until its release callback runs. It holds no row lock or SQL transaction.+func acquireMySQLSyncDocumentLock(ctx context.Context, pool *sql.DB, key string) (func(), error) {+ lockCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)+ defer cancel()+ conn, err := pool.Conn(lockCtx)+ if err != nil {+ return nil, err+ }+ var granted sql.NullInt64+ err = conn.QueryRowContext(lockCtx, "SELECT GET_LOCK(?, ?)", key, int64(cleanupBatchTimeout/time.Second)).Scan(&granted)+ if err != nil || !granted.Valid || granted.Int64 != 1 {+ _ = conn.Close()+ if err != nil {+ return nil, err+ }+ return nil, fmt.Errorf("could not acquire sync document lock %s", key)+ }+ return func() {+ defer conn.Close()+ releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupBatchTimeout)+ defer cancel()+ var released sql.NullInt64+ if err := conn.QueryRowContext(releaseCtx, "SELECT RELEASE_LOCK(?)", key).Scan(&released); err != nil || !released.Valid || released.Int64 != 1 {+ common.Warn(fmt.Sprintf("release sync document lock %s failed: %v", key, err))+ }+ }, nil+}diff --git a/internal/service/document/sync_document_lock_test.go b/internal/service/document/sync_document_lock_test.gonew file mode 100644index 0000000000..f054fddff5--- /dev/null+++ b/internal/service/document/sync_document_lock_test.go@@ -0,0 +1,85 @@+package document++import (+ "context"+ "errors"+ "testing"++ "github.com/DATA-DOG/go-sqlmock"+)++func TestMySQLSyncDocumentLockAcquisitionOutcomes(t *testing.T) {+ for _, tc := range []struct {+ name string+ value interface{}+ err error+ }{+ {"timeout", int64(0), nil},+ {"null", nil, nil},+ {"query error", nil, errors.New("lock query failed")},+ } {+ t.Run(tc.name, func(t *testing.T) {+ pool, mock, err := sqlmock.New()+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { pool.Close() })+ query := mock.ExpectQuery(`SELECT GET_LOCK\(\?, \?\)`).WithArgs("sync-document:test", int64(30))+ if tc.err != nil {+ query.WillReturnError(tc.err)+ } else {+ query.WillReturnRows(sqlmock.NewRows([]string{"granted"}).AddRow(tc.value))+ }+ unlock, err := acquireMySQLSyncDocumentLock(t.Context(), pool, "sync-document:test")+ if err == nil || unlock != nil {+ t.Fatalf("failed acquisition = %v, unlock = %v", err, unlock != nil)+ }+ if tc.err != nil && !errors.Is(err, tc.err) {+ t.Fatalf("query error = %v", err)+ }+ if err := mock.ExpectationsWereMet(); err != nil {+ t.Fatal(err)+ }+ })+ }+}++func TestMySQLSyncDocumentLockReleaseAfterCancellation(t *testing.T) {+ for _, releaseFails := range []bool{false, true} {+ name := "release-success"+ if releaseFails {+ name = "release-error"+ }+ t.Run(name, func(t *testing.T) {+ pool, mock, err := sqlmock.New()+ if err != nil {+ t.Fatal(err)+ }+ pool.SetMaxOpenConns(1)+ mock.ExpectQuery(`SELECT GET_LOCK\(\?, \?\)`).WithArgs("sync-document:test", int64(30)).+ WillReturnRows(sqlmock.NewRows([]string{"granted"}).AddRow(1))+ release := mock.ExpectQuery(`SELECT RELEASE_LOCK\(\?\)`).WithArgs("sync-document:test")+ if releaseFails {+ release.WillReturnError(errors.New("release unavailable"))+ } else {+ release.WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1))+ }+ mock.ExpectClose()+ ctx, cancel := context.WithCancel(t.Context())+ unlock, err := acquireMySQLSyncDocumentLock(ctx, pool, "sync-document:test")+ if err != nil {+ cancel()+ t.Fatal(err)+ }+ cancel()+ unlock()+ // The owning callback closes this dedicated pool even on release error.+ if err := pool.Close(); err != nil {+ t.Fatal(err)+ }+ if err := mock.ExpectationsWereMet(); err != nil {+ t.Fatal(err)+ }+ })+ }+}diff --git a/internal/service/document/sync_document_test.go b/internal/service/document/sync_document_test.gonew file mode 100644index 0000000000..477b2a5148--- /dev/null+++ b/internal/service/document/sync_document_test.go@@ -0,0 +1,402 @@+package document++import (+ "context"+ "errors"+ "strings"+ "sync"+ "testing"+ "time"++ "ragflow/internal/common"+ "ragflow/internal/dao"+ "ragflow/internal/entity"+ "ragflow/internal/service"+ "ragflow/internal/storage"+ syncerconnector "ragflow/internal/syncer/connector"++ "gorm.io/gorm"+)++type syncMetadataFaultEngine struct {+ fakeChatDocEngine+ err error+ onUpdate func() error+}++func (e *syncMetadataFaultEngine) UpdateMetadata(context.Context, string, string, map[string]interface{}, string) error {+ if e.onUpdate != nil {+ return e.onUpdate()+ }+ return e.err+}++func setupSyncDocumentTest(t *testing.T, existing bool) (*DocumentService, *gorm.DB, *fakeUploadStorage, service.DocumentUpsertInput) {+ t.Helper()+ db := setupServiceTestDB(t)+ pushServiceDB(t, db)+ insertTestKB(t, "kb-sync", "tenant-1", 0, 0, 0)+ kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-sync")+ if err != nil {+ t.Fatal(err)+ }+ store := newFakeUploadStorage()+ factory := storage.GetStorageFactory()+ original := factory.GetStorage()+ factory.SetStorage(store)+ t.Cleanup(func() { factory.SetStorage(original) })+ svc := testDocumentService(t)+ input := service.DocumentUpsertInput{+ TaskContext: service.SyncTaskContext{Knowledgebase: *kb, Connector: entity.Connector{TenantID: "tenant-1"}},+ SourceType: "github_connector-1",+ DocumentID: "doc-sync",+ SourceDocument: syncerconnector.SourceDocument{+ SourceID: "source-1", SemanticIdentifier: "source", Extension: ".txt",+ Fingerprint: "new-hash", Blob: []byte("new content"),+ },+ }+ if existing {+ old := input+ old.SourceDocument.Fingerprint = "old-hash"+ old.SourceDocument.Blob = []byte("old content")+ if _, err := svc.Upsert(t.Context(), old); err != nil {+ t.Fatalf("seed existing document: %v", err)+ }+ }+ return svc, db, store, input+}++func assertSyncDocumentState(t *testing.T, db *gorm.DB, store *fakeUploadStorage, input service.DocumentUpsertInput, hash string) *entity.Document {+ t.Helper()+ doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, input.DocumentID)+ if err != nil {+ t.Fatal(err)+ }+ if doc.ContentHash == nil || *doc.ContentHash != hash {+ t.Fatalf("fingerprint = %v, want %q", doc.ContentHash, hash)+ }+ if doc.Location == nil {+ t.Fatal("document has no storage location")+ }+ blob, err := store.Get(t.Context(), doc.KbID, *doc.Location)+ if err != nil || string(blob) != string(input.SourceDocument.Blob) {+ t.Fatalf("published blob = %q, err = %v", blob, err)+ }+ return doc+}++func TestSyncDocumentDownstreamFailurePreservesPublishedBlobAndRetries(t *testing.T) {+ for _, existing := range []bool{false, true} {+ for _, fault := range []string{"metadata", "fingerprint", "file-manager"} {+ if !existing && fault == "file-manager" {+ continue // Insert link failures roll the document back; covered separately.+ }+ name := "insert/" + fault+ if existing {+ name = "update/" + fault+ }+ t.Run(name, func(t *testing.T) {+ svc, db, store, input := setupSyncDocumentTest(t, existing)+ faultErr := errors.New("injected " + fault + " failure")+ engine := &syncMetadataFaultEngine{}+ if fault == "metadata" {+ input.SourceDocument.Metadata = map[string]interface{}{"category": "changed"}+ engine.err = faultErr+ svc.docEngine = engine+ svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)+ } else {+ if err := db.Callback().Update().Before("gorm:update").Register("sync-fault", func(tx *gorm.DB) {+ updates, _ := tx.Statement.Dest.(map[string]interface{})+ if fault == "file-manager" && tx.Statement.Table == "file" ||+ fault == "fingerprint" && tx.Statement.Table == "document" && updates["content_hash"] == "new-hash" {+ tx.AddError(faultErr)+ }+ }); err != nil {+ t.Fatal(err)+ }+ }+ if _, err := svc.Upsert(t.Context(), input); !errors.Is(err, faultErr) {+ t.Fatalf("Upsert error = %v, want injected failure", err)+ }+ assertSyncDocumentState(t, db, store, input, "")+ // Both runner-level and connector-level skip checks use this persisted hash.+ fingerprints, err := service.NewGormDocumentStore().ListFingerprintsBySourceType(t.Context(), input.TaskContext.Knowledgebase.ID, input.SourceType)+ if err != nil || fingerprints[input.DocumentID] != "" {+ t.Fatalf("pending fingerprint snapshot = %v, err = %v", fingerprints, err)+ }+ engine.err = nil+ if fault != "metadata" {+ if err := db.Callback().Update().Remove("sync-fault"); err != nil {+ t.Fatal(err)+ }+ }+ result, err := svc.Upsert(t.Context(), input)+ if err != nil || result.Action != service.DocumentActionUpdated {+ t.Fatalf("retry = %+v, err = %v", result, err)+ }+ doc := assertSyncDocumentState(t, db, store, input, "new-hash")+ location := *doc.Location+ count := len(store.objects)+ result, err = svc.Upsert(t.Context(), input)+ if err != nil || result.Action != service.DocumentActionSkipped {+ t.Fatalf("unchanged successful sync = %+v, err = %v", result, err)+ }+ if doc = assertSyncDocumentState(t, db, store, input, "new-hash"); *doc.Location != location || len(store.objects) != count {+ t.Fatal("unchanged successful sync rewrote the blob")+ }+ })+ }+ }+}++func TestSyncDocumentRunningParseRetryAndSourceRevert(t *testing.T) {+ for _, revert := range []bool{false, true} {+ name := "same-revision"+ if revert {+ name = "source-reverted"+ }+ t.Run(name, func(t *testing.T) {+ svc, db, store, input := setupSyncDocumentTest(t, true)+ input.AutoParse = true+ insertTestIngestionTaskWithStatus(t, "old-task", "tenant-1", input.DocumentID, "kb-sync", common.RUNNING)+ for attempt := 0; attempt < 2; attempt++ {+ if _, err := svc.Upsert(t.Context(), input); err == nil || !strings.Contains(err.Error(), common.RUNNING) {+ t.Fatalf("running parse Upsert = %v", err)+ }+ assertSyncDocumentState(t, db, store, input, "")+ }+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "old-task").Update("status", common.COMPLETED).Error; err != nil {+ t.Fatal(err)+ }+ if revert {+ input.SourceDocument.Fingerprint = "old-hash"+ input.SourceDocument.Blob = []byte("old content")+ }+ svc.purgeTaskState = func(context.Context, string) error { return nil }+ publisher := &recordingTaskPublisher{}+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)+ result, err := svc.Upsert(t.Context(), input)+ if err != nil || result.Action != service.DocumentActionUpdated {+ t.Fatalf("terminal task retry = %+v, err = %v", result, err)+ }+ assertSyncDocumentState(t, db, store, input, input.SourceDocument.Fingerprint)+ if len(publisher.messages) != 1 || publisher.messages[0].TaskID == "old-task" {+ t.Fatalf("replacement parse publications = %#v", publisher.messages)+ }+ if result, err = svc.Upsert(t.Context(), input); err != nil || result.Action != service.DocumentActionSkipped || len(publisher.messages) != 1 {+ t.Fatalf("unchanged parse retry = %+v, err = %v, publications = %d", result, err, len(publisher.messages))+ }+ })+ }+}++func TestSyncDocumentInsertLinkRollbackCleanup(t *testing.T) {+ for _, rollbackFails := range []bool{false, true} {+ name := "rollback-success"+ if rollbackFails {+ name = "rollback-failure"+ }+ t.Run(name, func(t *testing.T) {+ svc, db, store, input := setupSyncDocumentTest(t, false)+ if err := db.Callback().Create().Before("gorm:create").Register("link-fault", func(tx *gorm.DB) {+ if file, ok := tx.Statement.Dest.(*entity.File); ok && file.Type != "folder" {+ tx.AddError(errors.New("file link failed"))+ }+ }); err != nil {+ t.Fatal(err)+ }+ if rollbackFails {+ if err := db.Callback().Delete().Before("gorm:delete").Register("rollback-fault", func(tx *gorm.DB) {+ if tx.Statement.Table == "document" {+ tx.AddError(errors.New("rollback failed"))+ }+ }); err != nil {+ t.Fatal(err)+ }+ }+ if _, err := svc.Upsert(t.Context(), input); err == nil {+ t.Fatal("expected file link failure")+ }+ if rollbackFails {+ assertSyncDocumentState(t, db, store, input, "")+ } else if len(store.objects) != 0 {+ t.Fatalf("rolled-back document left %d unreferenced blobs", len(store.objects))+ }+ })+ }+}++func TestSyncDocumentConcurrentPublicationCannotAcknowledgeAnotherRevision(t *testing.T) {+ svc, db, store, input := setupSyncDocumentTest(t, true)+ input.SourceDocument.Metadata = map[string]interface{}{"category": "changed"}+ replacementLocation := "sync/concurrent.txt"+ engine := &syncMetadataFaultEngine{onUpdate: func() error {+ if err := store.Put(t.Context(), "kb-sync", replacementLocation, []byte("another revision")); err != nil {+ return err+ }+ return db.Model(&entity.Document{}).Where("id = ?", input.DocumentID).+ Updates(map[string]interface{}{"location": replacementLocation, "content_hash": ""}).Error+ }}+ svc.docEngine = engine+ svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)+ if _, err := svc.Upsert(t.Context(), input); err == nil || !strings.Contains(err.Error(), "changed before") {+ t.Fatalf("concurrent publication Upsert = %v", err)+ }+ input.SourceDocument.Blob = []byte("another revision")+ doc := assertSyncDocumentState(t, db, store, input, "")+ if *doc.Location != replacementLocation {+ t.Fatalf("location = %q, want concurrent revision", *doc.Location)+ }+}++func TestSyncDocumentCleanupPreservesBlobWhenReferenceLookupFails(t *testing.T) {+ svc, db, store, input := setupSyncDocumentTest(t, true)+ store.afterPut = func() {+ if err := db.Callback().Query().Before("gorm:query").Register("lookup-fault", func(tx *gorm.DB) {+ if tx.Statement.Table == "document" {+ tx.AddError(errors.New("reference lookup unavailable"))+ }+ }); err != nil {+ t.Fatal(err)+ }+ }+ if err := db.Callback().Update().Before("gorm:update").Register("update-fault", func(tx *gorm.DB) {+ if tx.Statement.Table == "document" {+ tx.AddError(errors.New("update unavailable"))+ }+ }); err != nil {+ t.Fatal(err)+ }+ if _, err := svc.Upsert(t.Context(), input); err == nil {+ t.Fatal("expected update failure")+ }+ if len(store.objects) != 2 {+ t.Fatalf("unknown publication state removed a blob: objects = %d", len(store.objects))+ }+}++func TestSyncDocumentSerializesMetadataAcrossRevisions(t *testing.T) {+ for _, existing := range []bool{false, true} {+ name := "new-document"+ if existing {+ name = "existing-document"+ }+ t.Run(name, func(t *testing.T) {+ first, db, store, input := setupSyncDocumentTest(t, existing)+ second := testDocumentService(t)+ firstEntered := make(chan struct{})+ releaseFirst := make(chan struct{})+ var releaseOnce sync.Once+ release := func() { releaseOnce.Do(func() { close(releaseFirst) }) }+ defer release()+ var mu sync.Mutex+ metadata := ""+ firstEngine := &syncMetadataFaultEngine{onUpdate: func() error {+ close(firstEntered)+ <-releaseFirst+ mu.Lock()+ metadata = "first"+ mu.Unlock()+ return nil+ }}+ secondEngine := &syncMetadataFaultEngine{onUpdate: func() error {+ mu.Lock()+ metadata = "second"+ mu.Unlock()+ return nil+ }}+ first.docEngine = firstEngine+ first.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), firstEngine)+ second.docEngine = secondEngine+ second.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), secondEngine)+ input.SourceDocument.Metadata = map[string]interface{}{"revision": "first"}+ firstResult := make(chan error, 1)+ go func() { _, err := first.Upsert(t.Context(), input); firstResult <- err }()+ select {+ case <-firstEntered:+ case <-time.After(5 * time.Second):+ t.Fatal("first revision did not reach metadata")+ }+ next := input+ next.SourceDocument.Fingerprint = "second-hash"+ next.SourceDocument.Blob = []byte("second content")+ next.SourceDocument.Metadata = map[string]interface{}{"revision": "second"}+ secondStarted := make(chan struct{})+ secondResult := make(chan error, 1)+ go func() {+ close(secondStarted)+ _, err := second.Upsert(t.Context(), next)+ secondResult <- err+ }()+ <-secondStarted+ select {+ case err := <-secondResult:+ t.Fatalf("second revision escaped the first revision's metadata barrier: %v", err)+ case <-time.After(100 * time.Millisecond):+ }+ release()+ for _, result := range []<-chan error{firstResult, secondResult} {+ select {+ case err := <-result:+ if err != nil {+ t.Fatal(err)+ }+ case <-time.After(5 * time.Second):+ t.Fatal("serialized revision did not finish")+ }+ }+ assertSyncDocumentState(t, db, store, next, "second-hash")+ mu.Lock()+ defer mu.Unlock()+ if metadata != "second" {+ t.Fatalf("acknowledged second revision has stale metadata %q", metadata)+ }+ })+ }+}++func TestSyncDocumentAcknowledgesCompletedWorkAfterCancellation(t *testing.T) {+ svc, db, store, input := setupSyncDocumentTest(t, true)+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ engine := &syncMetadataFaultEngine{onUpdate: func() error { cancel(); return nil }}+ svc.docEngine = engine+ svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)+ input.SourceDocument.Metadata = map[string]interface{}{"revision": "new"}+ result, err := svc.Upsert(ctx, input)+ if err != nil || result.Action != service.DocumentActionUpdated {+ t.Fatalf("completed dependent work was not acknowledged: result = %+v err = %v", result, err)+ }+ assertSyncDocumentState(t, db, store, input, "new-hash")+}++func TestSyncDocumentAcknowledgesParseEnqueueAfterCancellation(t *testing.T) {+ svc, db, store, input := setupSyncDocumentTest(t, true)+ ctx, cancel := context.WithCancel(t.Context())+ defer cancel()+ publisher := &recordingTaskPublisher{}+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)+ input.AutoParse = true+ // Cancel at the final fingerprint write, after StartParseDocuments has+ // completed both publication and its scheduled-state bookkeeping.+ if err := db.Callback().Update().Before("gorm:update").Register("cancel-before-ack", func(tx *gorm.DB) {+ updates, _ := tx.Statement.Dest.(map[string]interface{})+ if tx.Statement.Table == "document" && updates["content_hash"] == input.SourceDocument.Fingerprint {+ cancel()+ }+ }); err != nil {+ t.Fatal(err)+ }+ result, err := svc.Upsert(ctx, input)+ if err != nil || result.Action != service.DocumentActionUpdated {+ t.Fatalf("successful enqueue was not acknowledged: result = %+v err = %v", result, err)+ }+ assertSyncDocumentState(t, db, store, input, "new-hash")+ if len(publisher.messages) != 1 {+ t.Fatalf("parse publications = %d, want 1", len(publisher.messages))+ }+ result, err = svc.Upsert(t.Context(), input)+ if err != nil || result.Action != service.DocumentActionSkipped || len(publisher.messages) != 1 {+ t.Fatalf("completed revision retried enqueue: result = %+v err = %v publications = %d", result, err, len(publisher.messages))+ }+}diff --git a/internal/service/sync_document_id_resolver.go b/internal/service/sync_document_id_resolver.goindex 68edbf66b4..336a44f4f6 100644--- a/internal/service/sync_document_id_resolver.go+++ b/internal/service/sync_document_id_resolver.go@@ -75,12 +75,7 @@ func (r *DocumentIDResolver) Resolve(ctx context.Context, kbID, connectorID, souif _, ok := fingerprints[legacyID]; ok {docID = legacyID}- stored := fingerprints[legacyID]- if stored == "" {- stored = fingerprints[newID]- }-- return ResolvedDocumentID{DocID: docID, LegacyID: legacyID, NewID: newID, StoredFingerprint: stored}, nil+ return ResolvedDocumentID{DocID: docID, LegacyID: legacyID, NewID: newID, StoredFingerprint: fingerprints[docID]}, nil}// GormDocumentStore reads documents from the current GORM database.diff --git a/internal/service/sync_document_id_resolver_test.go b/internal/service/sync_document_id_resolver_test.gonew file mode 100644index 0000000000..f669385316--- /dev/null+++ b/internal/service/sync_document_id_resolver_test.go@@ -0,0 +1,39 @@+package service++import (+ "context"+ "testing"+)++type resolverFingerprintStore struct {+ DocumentStore+ fingerprints map[string]string+}++func (s resolverFingerprintStore) GetFingerprintsByIDs(context.Context, string, string, []string) (map[string]string, error) {+ return s.fingerprints, nil+}++func TestDocumentIDResolverUsesChosenDocumentsFingerprint(t *testing.T) {+ legacyID := Hash128("connector:source")+ newID := Hash128("kb:connector:source")+ for _, tc := range []struct {+ name string+ fingerprints map[string]string+ wantID string+ wantHash string+ }{+ {"pending legacy", map[string]string{legacyID: "", newID: "completed-new-hash"}, legacyID, ""},+ {"completed legacy", map[string]string{legacyID: "legacy-hash", newID: "new-hash"}, legacyID, "legacy-hash"},+ {"new document", map[string]string{newID: "new-hash"}, newID, "new-hash"},+ {"missing document", map[string]string{}, newID, ""},+ } {+ t.Run(tc.name, func(t *testing.T) {+ resolver := NewDocumentIDResolver(resolverFingerprintStore{fingerprints: tc.fingerprints})+ got, err := resolver.Resolve(t.Context(), "kb", "connector", "github_connector", "source")+ if err != nil || got.DocID != tc.wantID || got.StoredFingerprint != tc.wantHash {+ t.Fatalf("resolved = %+v, err = %v; want id = %s hash = %q", got, err, tc.wantID, tc.wantHash)+ }+ })+ }+}