Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(sync): preserve published blobs and retry incomplete revisions (#20627)

Lucienedeabb021e39
diff --git a/internal/dao/mysql_session.go b/internal/dao/mysql_session.go
new file mode 100644
index 0000000000..e05b7faeed
--- /dev/null
+++ b/internal/dao/mysql_session.go
@@ -0,0 +1,38 @@
+package dao
+
+import (
+ "database/sql"
+ "fmt"
+
+ "gorm.io/driver/mysql"
+ "gorm.io/gorm"
+)
+
+// OpenIndependentMySQLDB opens a separate pool with the active connection
+// settings. Session locks must not reserve connections from the application
+// pool while their owners run database operations through that same pool.
+func OpenIndependentMySQLDB(db *gorm.DB) (*sql.DB, error) {
+ var config *mysql.Config
+ switch dialector := db.Dialector.(type) {
+ case migrationAwareDialector:
+ config = dialector.Config
+ case *mysql.Dialector:
+ config = dialector.Config
+ default:
+ return nil, fmt.Errorf("independent session requires a MySQL dialector")
+ }
+ if config == nil || config.DSN == "" {
+ return nil, fmt.Errorf("independent session requires a MySQL DSN")
+ }
+ driverName := config.DriverName
+ if driverName == "" {
+ driverName = "mysql"
+ }
+ pool, err := sql.Open(driverName, config.DSN)
+ if err != nil {
+ return nil, err
+ }
+ pool.SetMaxOpenConns(1)
+ pool.SetMaxIdleConns(0)
+ return pool, nil
+}
diff --git a/internal/dao/mysql_session_test.go b/internal/dao/mysql_session_test.go
new file mode 100644
index 0000000000..b01e3c1b1b
--- /dev/null
+++ b/internal/dao/mysql_session_test.go
@@ -0,0 +1,42 @@
+package dao
+
+import (
+ "testing"
+
+ "github.com/DATA-DOG/go-sqlmock"
+ "gorm.io/driver/mysql"
+ "gorm.io/gorm"
+)
+
+func TestIndependentMySQLDBSupportsMigrationDialector(t *testing.T) {
+ for _, wrapped := range []bool{false, true} {
+ name := "mysql"
+ if wrapped {
+ name = "migration wrapper"
+ }
+ t.Run(name, func(t *testing.T) {
+ shared, _, err := sqlmock.New()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { shared.Close() })
+ base := mysql.New(mysql.Config{DSN: "test:test@tcp(localhost:3306)/test?parseTime=True", Conn: shared, SkipInitializeWithVersion: true}).(*mysql.Dialector)
+ var dialect gorm.Dialector = base
+ if wrapped {
+ dialect = migrationAwareDialector{Dialector: base}
+ }
+ db, err := gorm.Open(dialect, &gorm.Config{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ independent, err := OpenIndependentMySQLDB(db)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer independent.Close()
+ if independent == shared || independent.Stats().MaxOpenConnections != 1 {
+ t.Fatal("session lock must have its own bounded connection pool")
+ }
+ })
+ }
+}
diff --git a/internal/service/document/sync_document.go b/internal/service/document/sync_document.go
index 6c38361d49..5395688c09 100644
--- a/internal/service/document/sync_document.go
+++ b/internal/service/document/sync_document.go
@@ -63,6 +63,14 @@ func (s *DocumentService) Upsert(ctx context.Context, input service.DocumentUpse
if contentHash == "" {
contentHash = contentHashHex(input.SourceDocument.Blob)
}
+ // Serialize publication and all dependent side effects, including the
+ // unchanged-revision path. A final fingerprint CAS cannot undo stale
+ // metadata written by an earlier concurrent revision.
+ unlock, err := acquireSyncDocumentLock(ctx, input.DocumentID)
+ if err != nil {
+ return service.DocumentUpsertResult{}, err
+ }
+ defer unlock()
// if the 'file' is existing
existing, err := s.documentDAO.GetByID(ctx, dao.DB, input.DocumentID)
@@ -81,7 +89,18 @@ func (s *DocumentService) Upsert(ctx context.Context, input service.DocumentUpse
cleanupLocation := true
defer func() {
if cleanupLocation {
- _ = storageImpl.Remove(context.WithoutCancel(ctx), kb.ID, location)
+ cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupBatchTimeout)
+ defer cancel()
+ // A downstream failure can leave the staged key published. Only
+ // remove it when the database confirms it is no longer referenced.
+ current, lookupErr := s.documentDAO.GetByID(cleanupCtx, dao.DB, input.DocumentID)
+ if lookupErr != nil && !errorsIsRecordNotFound(lookupErr) {
+ return
+ }
+ if current != nil && current.Location != nil && *current.Location == location {
+ return
+ }
+ _ = storageImpl.Remove(cleanupCtx, kb.ID, location)
}
}()
if err = storageImpl.Put(ctx, kb.ID, location, input.SourceDocument.Blob); err != nil {
@@ -118,12 +137,14 @@ func (s *DocumentService) insertSyncDocument(ctx context.Context, input service.
parserID, parserConfig := resolveDocumentParser(ctx, kb, filename, utility.FileType(filetype), kb.ParserConfig)
doc := s.newDatasetDocument(kb, tenantID, filename, "", filetype, parserID, parserConfig, input.SourceType, int64(len(input.SourceDocument.Blob)), input.SourceDocument.Blob)
doc.ID = input.DocumentID
+ // A fingerprint is acknowledged only after all dependent work succeeds.
+ doc.ContentHash = nil
// put 'file' in mysql `document`
if err = s.InsertDocument(doc); err != nil {
return service.DocumentUpsertResult{}, err
}
- if err = s.publishSyncDocument(ctx, doc, location, contentHash); err != nil {
+ if err = s.publishSyncDocument(ctx, doc, location); err != nil {
return service.DocumentUpsertResult{}, s.rollbackAddFileFromKBError(ctx, doc, kb.ID, err)
}
// link 'file' 2 'file_system'
@@ -131,7 +152,7 @@ func (s *DocumentService) insertSyncDocument(ctx context.Context, input service.
return service.DocumentUpsertResult{}, s.rollbackAddFileFromKBError(ctx, doc, kb.ID, err)
}
// write metadata and do `auto_parse` (or not)
- if err = s.afterSyncDocumentUpsert(ctx, input, doc, false); err != nil {
+ if err = s.afterSyncDocumentUpsert(ctx, input, doc, false, contentHash); err != nil {
return service.DocumentUpsertResult{}, err
}
@@ -158,7 +179,7 @@ func (s *DocumentService) updateSyncDocument(ctx context.Context, input service.
doc.Suffix = suffix
doc.SourceType = input.SourceType
- if err := s.publishSyncDocument(ctx, doc, location, contentHash); err != nil {
+ if err := s.publishSyncDocument(ctx, doc, location); err != nil {
return service.DocumentUpsertResult{}, err
}
// update new updated file
@@ -166,14 +187,17 @@ func (s *DocumentService) updateSyncDocument(ctx context.Context, input service.
return service.DocumentUpsertResult{}, err
}
// write metadata and do `auto_parse`
- if err := s.afterSyncDocumentUpsert(ctx, input, doc, true); err != nil {
+ if err := s.afterSyncDocumentUpsert(ctx, input, doc, true, contentHash); err != nil {
return service.DocumentUpsertResult{}, err
}
return service.DocumentUpsertResult{DocID: doc.ID, Action: service.DocumentActionUpdated}, nil
}
// publishSyncDocument publishes the staged object key after the document row exists.
-func (s *DocumentService) publishSyncDocument(ctx context.Context, doc *entity.Document, location, contentHash string) error {
+func (s *DocumentService) publishSyncDocument(ctx context.Context, doc *entity.Document, location string) error {
+ // Invalidate the old fingerprint atomically with the location change so
+ // runners and connectors cannot skip an incompletely processed revision.
+ contentHash := ""
if err := s.documentDAO.UpdateByID(ctx, dao.DB, doc.ID, map[string]interface{}{
"location": location,
"content_hash": contentHash,
@@ -186,7 +210,7 @@ func (s *DocumentService) publishSyncDocument(ctx context.Context, doc *entity.D
}
// afterSyncDocumentUpsert writes metadata and optionally enqueues parsing.
-func (s *DocumentService) afterSyncDocumentUpsert(ctx context.Context, input service.DocumentUpsertInput, doc *entity.Document, rerun bool) error {
+func (s *DocumentService) afterSyncDocumentUpsert(ctx context.Context, input service.DocumentUpsertInput, doc *entity.Document, rerun bool, contentHash string) error {
// write metadata
if len(input.SourceDocument.Metadata) > 0 && s.docEngine != nil {
if err := s.SetDocumentMetadata(ctx, doc.ID, input.SourceDocument.Metadata); err != nil {
@@ -194,10 +218,26 @@ func (s *DocumentService) afterSyncDocumentUpsert(ctx context.Context, input ser
}
}
// do auto_parse if enabled
- if !input.AutoParse {
- return nil
+ if input.AutoParse {
+ if err := s.StartParseDocuments(ctx, doc, &input.TaskContext.Knowledgebase, input.TaskContext.Connector.TenantID, StartParseOptions{RerunWithDelete: rerun}); err != nil {
+ return err
+ }
+ }
+ // Defend against non-sync writers replacing the document while metadata
+ // or parsing was being prepared. Never acknowledge another blob's hash.
+ ackCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupBatchTimeout)
+ defer cancel()
+ result := dao.DB.WithContext(ackCtx).Model(&entity.Document{}).
+ Where("id = ? AND location = ?", doc.ID, doc.Location).
+ Updates(map[string]interface{}{"content_hash": contentHash})
+ if result.Error != nil {
+ return result.Error
+ }
+ if result.RowsAffected == 0 {
+ return fmt.Errorf("synced document %s changed before its fingerprint was committed", doc.ID)
}
- return s.StartParseDocuments(ctx, doc, &input.TaskContext.Knowledgebase, input.TaskContext.Connector.TenantID, StartParseOptions{RerunWithDelete: rerun})
+ doc.ContentHash = &contentHash
+ return nil
}
// ensureSyncDocumentPostWrite retries dependent work before an unchanged document is skipped.
diff --git a/internal/service/document/sync_document_lock.go b/internal/service/document/sync_document_lock.go
new file mode 100644
index 0000000000..626edef6fc
--- /dev/null
+++ b/internal/service/document/sync_document_lock.go
@@ -0,0 +1,68 @@
+package document
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/service"
+)
+
+// acquireSyncDocumentLock serializes sync side effects across MySQL sessions,
+// including for documents that have not been inserted yet. SQLite uses the
+// process lock only for the default in-memory unit fixtures.
+func acquireSyncDocumentLock(ctx context.Context, docID string) (func(), error) {
+ key := "sync-document:" + service.Hash128(docID)
+ if dao.DB.Dialector.Name() == "sqlite" {
+ // SQLite is used by the in-memory service unit fixtures. Its lock key
+ // must differ from the parse lock acquired by StartParseDocuments.
+ return lockDocumentParse(key), nil
+ }
+ pool, err := dao.OpenIndependentMySQLDB(dao.DB)
+ if err != nil {
+ return nil, err
+ }
+ unlock, err := acquireMySQLSyncDocumentLock(ctx, pool, key)
+ if err != nil {
+ _ = pool.Close()
+ return nil, err
+ }
+ return func() {
+ // Closing the dedicated session also releases a named lock if the
+ // explicit release fails; it is never returned to the application pool.
+ defer pool.Close()
+ unlock()
+ }, nil
+}
+
+// acquireMySQLSyncDocumentLock owns a named lock on one dedicated connection
+// until its release callback runs. It holds no row lock or SQL transaction.
+func acquireMySQLSyncDocumentLock(ctx context.Context, pool *sql.DB, key string) (func(), error) {
+ lockCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)
+ defer cancel()
+ conn, err := pool.Conn(lockCtx)
+ if err != nil {
+ return nil, err
+ }
+ var granted sql.NullInt64
+ err = conn.QueryRowContext(lockCtx, "SELECT GET_LOCK(?, ?)", key, int64(cleanupBatchTimeout/time.Second)).Scan(&granted)
+ if err != nil || !granted.Valid || granted.Int64 != 1 {
+ _ = conn.Close()
+ if err != nil {
+ return nil, err
+ }
+ return nil, fmt.Errorf("could not acquire sync document lock %s", key)
+ }
+ return func() {
+ defer conn.Close()
+ releaseCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupBatchTimeout)
+ defer cancel()
+ var released sql.NullInt64
+ if err := conn.QueryRowContext(releaseCtx, "SELECT RELEASE_LOCK(?)", key).Scan(&released); err != nil || !released.Valid || released.Int64 != 1 {
+ common.Warn(fmt.Sprintf("release sync document lock %s failed: %v", key, err))
+ }
+ }, nil
+}
diff --git a/internal/service/document/sync_document_lock_test.go b/internal/service/document/sync_document_lock_test.go
new file mode 100644
index 0000000000..f054fddff5
--- /dev/null
+++ b/internal/service/document/sync_document_lock_test.go
@@ -0,0 +1,85 @@
+package document
+
+import (
+ "context"
+ "errors"
+ "testing"
+
+ "github.com/DATA-DOG/go-sqlmock"
+)
+
+func TestMySQLSyncDocumentLockAcquisitionOutcomes(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ value interface{}
+ err error
+ }{
+ {"timeout", int64(0), nil},
+ {"null", nil, nil},
+ {"query error", nil, errors.New("lock query failed")},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ pool, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { pool.Close() })
+ query := mock.ExpectQuery(`SELECT GET_LOCK\(\?, \?\)`).WithArgs("sync-document:test", int64(30))
+ if tc.err != nil {
+ query.WillReturnError(tc.err)
+ } else {
+ query.WillReturnRows(sqlmock.NewRows([]string{"granted"}).AddRow(tc.value))
+ }
+ unlock, err := acquireMySQLSyncDocumentLock(t.Context(), pool, "sync-document:test")
+ if err == nil || unlock != nil {
+ t.Fatalf("failed acquisition = %v, unlock = %v", err, unlock != nil)
+ }
+ if tc.err != nil && !errors.Is(err, tc.err) {
+ t.Fatalf("query error = %v", err)
+ }
+ if err := mock.ExpectationsWereMet(); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
+
+func TestMySQLSyncDocumentLockReleaseAfterCancellation(t *testing.T) {
+ for _, releaseFails := range []bool{false, true} {
+ name := "release-success"
+ if releaseFails {
+ name = "release-error"
+ }
+ t.Run(name, func(t *testing.T) {
+ pool, mock, err := sqlmock.New()
+ if err != nil {
+ t.Fatal(err)
+ }
+ pool.SetMaxOpenConns(1)
+ mock.ExpectQuery(`SELECT GET_LOCK\(\?, \?\)`).WithArgs("sync-document:test", int64(30)).
+ WillReturnRows(sqlmock.NewRows([]string{"granted"}).AddRow(1))
+ release := mock.ExpectQuery(`SELECT RELEASE_LOCK\(\?\)`).WithArgs("sync-document:test")
+ if releaseFails {
+ release.WillReturnError(errors.New("release unavailable"))
+ } else {
+ release.WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1))
+ }
+ mock.ExpectClose()
+ ctx, cancel := context.WithCancel(t.Context())
+ unlock, err := acquireMySQLSyncDocumentLock(ctx, pool, "sync-document:test")
+ if err != nil {
+ cancel()
+ t.Fatal(err)
+ }
+ cancel()
+ unlock()
+ // The owning callback closes this dedicated pool even on release error.
+ if err := pool.Close(); err != nil {
+ t.Fatal(err)
+ }
+ if err := mock.ExpectationsWereMet(); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+}
diff --git a/internal/service/document/sync_document_test.go b/internal/service/document/sync_document_test.go
new file mode 100644
index 0000000000..477b2a5148
--- /dev/null
+++ b/internal/service/document/sync_document_test.go
@@ -0,0 +1,402 @@
+package document
+
+import (
+ "context"
+ "errors"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "ragflow/internal/common"
+ "ragflow/internal/dao"
+ "ragflow/internal/entity"
+ "ragflow/internal/service"
+ "ragflow/internal/storage"
+ syncerconnector "ragflow/internal/syncer/connector"
+
+ "gorm.io/gorm"
+)
+
+type syncMetadataFaultEngine struct {
+ fakeChatDocEngine
+ err error
+ onUpdate func() error
+}
+
+func (e *syncMetadataFaultEngine) UpdateMetadata(context.Context, string, string, map[string]interface{}, string) error {
+ if e.onUpdate != nil {
+ return e.onUpdate()
+ }
+ return e.err
+}
+
+func setupSyncDocumentTest(t *testing.T, existing bool) (*DocumentService, *gorm.DB, *fakeUploadStorage, service.DocumentUpsertInput) {
+ t.Helper()
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-sync", "tenant-1", 0, 0, 0)
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-sync")
+ if err != nil {
+ t.Fatal(err)
+ }
+ store := newFakeUploadStorage()
+ factory := storage.GetStorageFactory()
+ original := factory.GetStorage()
+ factory.SetStorage(store)
+ t.Cleanup(func() { factory.SetStorage(original) })
+ svc := testDocumentService(t)
+ input := service.DocumentUpsertInput{
+ TaskContext: service.SyncTaskContext{Knowledgebase: *kb, Connector: entity.Connector{TenantID: "tenant-1"}},
+ SourceType: "github_connector-1",
+ DocumentID: "doc-sync",
+ SourceDocument: syncerconnector.SourceDocument{
+ SourceID: "source-1", SemanticIdentifier: "source", Extension: ".txt",
+ Fingerprint: "new-hash", Blob: []byte("new content"),
+ },
+ }
+ if existing {
+ old := input
+ old.SourceDocument.Fingerprint = "old-hash"
+ old.SourceDocument.Blob = []byte("old content")
+ if _, err := svc.Upsert(t.Context(), old); err != nil {
+ t.Fatalf("seed existing document: %v", err)
+ }
+ }
+ return svc, db, store, input
+}
+
+func assertSyncDocumentState(t *testing.T, db *gorm.DB, store *fakeUploadStorage, input service.DocumentUpsertInput, hash string) *entity.Document {
+ t.Helper()
+ doc, err := dao.NewDocumentDAO().GetByID(t.Context(), db, input.DocumentID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if doc.ContentHash == nil || *doc.ContentHash != hash {
+ t.Fatalf("fingerprint = %v, want %q", doc.ContentHash, hash)
+ }
+ if doc.Location == nil {
+ t.Fatal("document has no storage location")
+ }
+ blob, err := store.Get(t.Context(), doc.KbID, *doc.Location)
+ if err != nil || string(blob) != string(input.SourceDocument.Blob) {
+ t.Fatalf("published blob = %q, err = %v", blob, err)
+ }
+ return doc
+}
+
+func TestSyncDocumentDownstreamFailurePreservesPublishedBlobAndRetries(t *testing.T) {
+ for _, existing := range []bool{false, true} {
+ for _, fault := range []string{"metadata", "fingerprint", "file-manager"} {
+ if !existing && fault == "file-manager" {
+ continue // Insert link failures roll the document back; covered separately.
+ }
+ name := "insert/" + fault
+ if existing {
+ name = "update/" + fault
+ }
+ t.Run(name, func(t *testing.T) {
+ svc, db, store, input := setupSyncDocumentTest(t, existing)
+ faultErr := errors.New("injected " + fault + " failure")
+ engine := &syncMetadataFaultEngine{}
+ if fault == "metadata" {
+ input.SourceDocument.Metadata = map[string]interface{}{"category": "changed"}
+ engine.err = faultErr
+ svc.docEngine = engine
+ svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)
+ } else {
+ if err := db.Callback().Update().Before("gorm:update").Register("sync-fault", func(tx *gorm.DB) {
+ updates, _ := tx.Statement.Dest.(map[string]interface{})
+ if fault == "file-manager" && tx.Statement.Table == "file" ||
+ fault == "fingerprint" && tx.Statement.Table == "document" && updates["content_hash"] == "new-hash" {
+ tx.AddError(faultErr)
+ }
+ }); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if _, err := svc.Upsert(t.Context(), input); !errors.Is(err, faultErr) {
+ t.Fatalf("Upsert error = %v, want injected failure", err)
+ }
+ assertSyncDocumentState(t, db, store, input, "")
+ // Both runner-level and connector-level skip checks use this persisted hash.
+ fingerprints, err := service.NewGormDocumentStore().ListFingerprintsBySourceType(t.Context(), input.TaskContext.Knowledgebase.ID, input.SourceType)
+ if err != nil || fingerprints[input.DocumentID] != "" {
+ t.Fatalf("pending fingerprint snapshot = %v, err = %v", fingerprints, err)
+ }
+ engine.err = nil
+ if fault != "metadata" {
+ if err := db.Callback().Update().Remove("sync-fault"); err != nil {
+ t.Fatal(err)
+ }
+ }
+ result, err := svc.Upsert(t.Context(), input)
+ if err != nil || result.Action != service.DocumentActionUpdated {
+ t.Fatalf("retry = %+v, err = %v", result, err)
+ }
+ doc := assertSyncDocumentState(t, db, store, input, "new-hash")
+ location := *doc.Location
+ count := len(store.objects)
+ result, err = svc.Upsert(t.Context(), input)
+ if err != nil || result.Action != service.DocumentActionSkipped {
+ t.Fatalf("unchanged successful sync = %+v, err = %v", result, err)
+ }
+ if doc = assertSyncDocumentState(t, db, store, input, "new-hash"); *doc.Location != location || len(store.objects) != count {
+ t.Fatal("unchanged successful sync rewrote the blob")
+ }
+ })
+ }
+ }
+}
+
+func TestSyncDocumentRunningParseRetryAndSourceRevert(t *testing.T) {
+ for _, revert := range []bool{false, true} {
+ name := "same-revision"
+ if revert {
+ name = "source-reverted"
+ }
+ t.Run(name, func(t *testing.T) {
+ svc, db, store, input := setupSyncDocumentTest(t, true)
+ input.AutoParse = true
+ insertTestIngestionTaskWithStatus(t, "old-task", "tenant-1", input.DocumentID, "kb-sync", common.RUNNING)
+ for attempt := 0; attempt < 2; attempt++ {
+ if _, err := svc.Upsert(t.Context(), input); err == nil || !strings.Contains(err.Error(), common.RUNNING) {
+ t.Fatalf("running parse Upsert = %v", err)
+ }
+ assertSyncDocumentState(t, db, store, input, "")
+ }
+ if err := db.Model(&entity.IngestionTask{}).Where("id = ?", "old-task").Update("status", common.COMPLETED).Error; err != nil {
+ t.Fatal(err)
+ }
+ if revert {
+ input.SourceDocument.Fingerprint = "old-hash"
+ input.SourceDocument.Blob = []byte("old content")
+ }
+ svc.purgeTaskState = func(context.Context, string) error { return nil }
+ publisher := &recordingTaskPublisher{}
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+ result, err := svc.Upsert(t.Context(), input)
+ if err != nil || result.Action != service.DocumentActionUpdated {
+ t.Fatalf("terminal task retry = %+v, err = %v", result, err)
+ }
+ assertSyncDocumentState(t, db, store, input, input.SourceDocument.Fingerprint)
+ if len(publisher.messages) != 1 || publisher.messages[0].TaskID == "old-task" {
+ t.Fatalf("replacement parse publications = %#v", publisher.messages)
+ }
+ if result, err = svc.Upsert(t.Context(), input); err != nil || result.Action != service.DocumentActionSkipped || len(publisher.messages) != 1 {
+ t.Fatalf("unchanged parse retry = %+v, err = %v, publications = %d", result, err, len(publisher.messages))
+ }
+ })
+ }
+}
+
+func TestSyncDocumentInsertLinkRollbackCleanup(t *testing.T) {
+ for _, rollbackFails := range []bool{false, true} {
+ name := "rollback-success"
+ if rollbackFails {
+ name = "rollback-failure"
+ }
+ t.Run(name, func(t *testing.T) {
+ svc, db, store, input := setupSyncDocumentTest(t, false)
+ if err := db.Callback().Create().Before("gorm:create").Register("link-fault", func(tx *gorm.DB) {
+ if file, ok := tx.Statement.Dest.(*entity.File); ok && file.Type != "folder" {
+ tx.AddError(errors.New("file link failed"))
+ }
+ }); err != nil {
+ t.Fatal(err)
+ }
+ if rollbackFails {
+ if err := db.Callback().Delete().Before("gorm:delete").Register("rollback-fault", func(tx *gorm.DB) {
+ if tx.Statement.Table == "document" {
+ tx.AddError(errors.New("rollback failed"))
+ }
+ }); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if _, err := svc.Upsert(t.Context(), input); err == nil {
+ t.Fatal("expected file link failure")
+ }
+ if rollbackFails {
+ assertSyncDocumentState(t, db, store, input, "")
+ } else if len(store.objects) != 0 {
+ t.Fatalf("rolled-back document left %d unreferenced blobs", len(store.objects))
+ }
+ })
+ }
+}
+
+func TestSyncDocumentConcurrentPublicationCannotAcknowledgeAnotherRevision(t *testing.T) {
+ svc, db, store, input := setupSyncDocumentTest(t, true)
+ input.SourceDocument.Metadata = map[string]interface{}{"category": "changed"}
+ replacementLocation := "sync/concurrent.txt"
+ engine := &syncMetadataFaultEngine{onUpdate: func() error {
+ if err := store.Put(t.Context(), "kb-sync", replacementLocation, []byte("another revision")); err != nil {
+ return err
+ }
+ return db.Model(&entity.Document{}).Where("id = ?", input.DocumentID).
+ Updates(map[string]interface{}{"location": replacementLocation, "content_hash": ""}).Error
+ }}
+ svc.docEngine = engine
+ svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)
+ if _, err := svc.Upsert(t.Context(), input); err == nil || !strings.Contains(err.Error(), "changed before") {
+ t.Fatalf("concurrent publication Upsert = %v", err)
+ }
+ input.SourceDocument.Blob = []byte("another revision")
+ doc := assertSyncDocumentState(t, db, store, input, "")
+ if *doc.Location != replacementLocation {
+ t.Fatalf("location = %q, want concurrent revision", *doc.Location)
+ }
+}
+
+func TestSyncDocumentCleanupPreservesBlobWhenReferenceLookupFails(t *testing.T) {
+ svc, db, store, input := setupSyncDocumentTest(t, true)
+ store.afterPut = func() {
+ if err := db.Callback().Query().Before("gorm:query").Register("lookup-fault", func(tx *gorm.DB) {
+ if tx.Statement.Table == "document" {
+ tx.AddError(errors.New("reference lookup unavailable"))
+ }
+ }); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := db.Callback().Update().Before("gorm:update").Register("update-fault", func(tx *gorm.DB) {
+ if tx.Statement.Table == "document" {
+ tx.AddError(errors.New("update unavailable"))
+ }
+ }); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := svc.Upsert(t.Context(), input); err == nil {
+ t.Fatal("expected update failure")
+ }
+ if len(store.objects) != 2 {
+ t.Fatalf("unknown publication state removed a blob: objects = %d", len(store.objects))
+ }
+}
+
+func TestSyncDocumentSerializesMetadataAcrossRevisions(t *testing.T) {
+ for _, existing := range []bool{false, true} {
+ name := "new-document"
+ if existing {
+ name = "existing-document"
+ }
+ t.Run(name, func(t *testing.T) {
+ first, db, store, input := setupSyncDocumentTest(t, existing)
+ second := testDocumentService(t)
+ firstEntered := make(chan struct{})
+ releaseFirst := make(chan struct{})
+ var releaseOnce sync.Once
+ release := func() { releaseOnce.Do(func() { close(releaseFirst) }) }
+ defer release()
+ var mu sync.Mutex
+ metadata := ""
+ firstEngine := &syncMetadataFaultEngine{onUpdate: func() error {
+ close(firstEntered)
+ <-releaseFirst
+ mu.Lock()
+ metadata = "first"
+ mu.Unlock()
+ return nil
+ }}
+ secondEngine := &syncMetadataFaultEngine{onUpdate: func() error {
+ mu.Lock()
+ metadata = "second"
+ mu.Unlock()
+ return nil
+ }}
+ first.docEngine = firstEngine
+ first.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), firstEngine)
+ second.docEngine = secondEngine
+ second.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), secondEngine)
+ input.SourceDocument.Metadata = map[string]interface{}{"revision": "first"}
+ firstResult := make(chan error, 1)
+ go func() { _, err := first.Upsert(t.Context(), input); firstResult <- err }()
+ select {
+ case <-firstEntered:
+ case <-time.After(5 * time.Second):
+ t.Fatal("first revision did not reach metadata")
+ }
+ next := input
+ next.SourceDocument.Fingerprint = "second-hash"
+ next.SourceDocument.Blob = []byte("second content")
+ next.SourceDocument.Metadata = map[string]interface{}{"revision": "second"}
+ secondStarted := make(chan struct{})
+ secondResult := make(chan error, 1)
+ go func() {
+ close(secondStarted)
+ _, err := second.Upsert(t.Context(), next)
+ secondResult <- err
+ }()
+ <-secondStarted
+ select {
+ case err := <-secondResult:
+ t.Fatalf("second revision escaped the first revision's metadata barrier: %v", err)
+ case <-time.After(100 * time.Millisecond):
+ }
+ release()
+ for _, result := range []<-chan error{firstResult, secondResult} {
+ select {
+ case err := <-result:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("serialized revision did not finish")
+ }
+ }
+ assertSyncDocumentState(t, db, store, next, "second-hash")
+ mu.Lock()
+ defer mu.Unlock()
+ if metadata != "second" {
+ t.Fatalf("acknowledged second revision has stale metadata %q", metadata)
+ }
+ })
+ }
+}
+
+func TestSyncDocumentAcknowledgesCompletedWorkAfterCancellation(t *testing.T) {
+ svc, db, store, input := setupSyncDocumentTest(t, true)
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ engine := &syncMetadataFaultEngine{onUpdate: func() error { cancel(); return nil }}
+ svc.docEngine = engine
+ svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)
+ input.SourceDocument.Metadata = map[string]interface{}{"revision": "new"}
+ result, err := svc.Upsert(ctx, input)
+ if err != nil || result.Action != service.DocumentActionUpdated {
+ t.Fatalf("completed dependent work was not acknowledged: result = %+v err = %v", result, err)
+ }
+ assertSyncDocumentState(t, db, store, input, "new-hash")
+}
+
+func TestSyncDocumentAcknowledgesParseEnqueueAfterCancellation(t *testing.T) {
+ svc, db, store, input := setupSyncDocumentTest(t, true)
+ ctx, cancel := context.WithCancel(t.Context())
+ defer cancel()
+ publisher := &recordingTaskPublisher{}
+ svc.ingestionTaskSvc.SetTaskPublisher(publisher)
+ input.AutoParse = true
+ // Cancel at the final fingerprint write, after StartParseDocuments has
+ // completed both publication and its scheduled-state bookkeeping.
+ if err := db.Callback().Update().Before("gorm:update").Register("cancel-before-ack", func(tx *gorm.DB) {
+ updates, _ := tx.Statement.Dest.(map[string]interface{})
+ if tx.Statement.Table == "document" && updates["content_hash"] == input.SourceDocument.Fingerprint {
+ cancel()
+ }
+ }); err != nil {
+ t.Fatal(err)
+ }
+ result, err := svc.Upsert(ctx, input)
+ if err != nil || result.Action != service.DocumentActionUpdated {
+ t.Fatalf("successful enqueue was not acknowledged: result = %+v err = %v", result, err)
+ }
+ assertSyncDocumentState(t, db, store, input, "new-hash")
+ if len(publisher.messages) != 1 {
+ t.Fatalf("parse publications = %d, want 1", len(publisher.messages))
+ }
+ result, err = svc.Upsert(t.Context(), input)
+ if err != nil || result.Action != service.DocumentActionSkipped || len(publisher.messages) != 1 {
+ t.Fatalf("completed revision retried enqueue: result = %+v err = %v publications = %d", result, err, len(publisher.messages))
+ }
+}
diff --git a/internal/service/sync_document_id_resolver.go b/internal/service/sync_document_id_resolver.go
index 68edbf66b4..336a44f4f6 100644
--- a/internal/service/sync_document_id_resolver.go
+++ b/internal/service/sync_document_id_resolver.go
@@ -75,12 +75,7 @@ func (r *DocumentIDResolver) Resolve(ctx context.Context, kbID, connectorID, sou
if _, ok := fingerprints[legacyID]; ok {
docID = legacyID
}
- stored := fingerprints[legacyID]
- if stored == "" {
- stored = fingerprints[newID]
- }
-
- return ResolvedDocumentID{DocID: docID, LegacyID: legacyID, NewID: newID, StoredFingerprint: stored}, nil
+ return ResolvedDocumentID{DocID: docID, LegacyID: legacyID, NewID: newID, StoredFingerprint: fingerprints[docID]}, nil
}
// GormDocumentStore reads documents from the current GORM database.
diff --git a/internal/service/sync_document_id_resolver_test.go b/internal/service/sync_document_id_resolver_test.go
new file mode 100644
index 0000000000..f669385316
--- /dev/null
+++ b/internal/service/sync_document_id_resolver_test.go
@@ -0,0 +1,39 @@
+package service
+
+import (
+ "context"
+ "testing"
+)
+
+type resolverFingerprintStore struct {
+ DocumentStore
+ fingerprints map[string]string
+}
+
+func (s resolverFingerprintStore) GetFingerprintsByIDs(context.Context, string, string, []string) (map[string]string, error) {
+ return s.fingerprints, nil
+}
+
+func TestDocumentIDResolverUsesChosenDocumentsFingerprint(t *testing.T) {
+ legacyID := Hash128("connector:source")
+ newID := Hash128("kb:connector:source")
+ for _, tc := range []struct {
+ name string
+ fingerprints map[string]string
+ wantID string
+ wantHash string
+ }{
+ {"pending legacy", map[string]string{legacyID: "", newID: "completed-new-hash"}, legacyID, ""},
+ {"completed legacy", map[string]string{legacyID: "legacy-hash", newID: "new-hash"}, legacyID, "legacy-hash"},
+ {"new document", map[string]string{newID: "new-hash"}, newID, "new-hash"},
+ {"missing document", map[string]string{}, newID, ""},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ resolver := NewDocumentIDResolver(resolverFingerprintStore{fingerprints: tc.fingerprints})
+ got, err := resolver.Resolve(t.Context(), "kb", "connector", "github_connector", "source")
+ if err != nil || got.DocID != tc.wantID || got.StoredFingerprint != tc.wantHash {
+ t.Fatalf("resolved = %+v, err = %v; want id = %s hash = %q", got, err, tc.wantID, tc.wantHash)
+ }
+ })
+ }
+}