Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(service): purge engine data from RemoveDocumentKeepFile (#20659) (#20768)

Harsh Kashyap730f12ba459a
diff --git a/internal/engine/elasticsearch/chunk.go b/internal/engine/elasticsearch/chunk.go
index 65b9b4efa6..e12965f467 100644
--- a/internal/engine/elasticsearch/chunk.go
+++ b/internal/engine/elasticsearch/chunk.go
@@ -1153,6 +1153,15 @@ func (e *Engine) DeleteChunks(ctx context.Context, condition map[string]interfac
deleted = int64(d)
}
+ // delete_by_query can answer 200 while only part of the work was done.
+ // Surface that so callers do not treat leftover chunks as deleted.
+ if timedOut, _ := result["timed_out"].(bool); timedOut {
+ return deleted, fmt.Errorf("elasticsearch delete by query timed out after deleting %d chunks from %s", deleted, fullIndexName)
+ }
+ if failures, ok := result["failures"].([]interface{}); ok && len(failures) > 0 {
+ return deleted, fmt.Errorf("elasticsearch delete by query reported %d failures after deleting %d chunks from %s", len(failures), deleted, fullIndexName)
+ }
+
common.Info("Successfully deleted chunks", zap.String("index_name", fullIndexName), zap.Int64("deleted_count", deleted))
return deleted, nil
}
diff --git a/internal/engine/elasticsearch/chunk_helpers_test.go b/internal/engine/elasticsearch/chunk_helpers_test.go
index ae5b3f89a2..d097303cb8 100644
--- a/internal/engine/elasticsearch/chunk_helpers_test.go
+++ b/internal/engine/elasticsearch/chunk_helpers_test.go
@@ -1014,6 +1014,78 @@ func TestMapMemoryMessageESUpdateFieldsRefreshesTokens(t *testing.T) {
assertEqual(t, doc["status_int"], 0)
}
+// TestDeleteChunksSurfacesPartialFailures guards against delete_by_query
+// answering 200 with timed_out or failures set while DeleteChunks reports nil.
+func TestDeleteChunksSurfacesPartialFailures(t *testing.T) {
+ cases := map[string]string{
+ "timed_out": `{"deleted":2,"timed_out":true,"failures":[]}`,
+ "failures": `{"deleted":2,"timed_out":false,"failures":[{"index":"ragflow_tenant","status":500}]}`,
+ }
+ for name, response := range cases {
+ t.Run(name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ if r.Method == http.MethodHead {
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(response))
+ }))
+ defer server.Close()
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatalf("new elasticsearch client: %v", err)
+ }
+ engine := &Engine{client: client}
+ deleted, err := engine.DeleteChunks(t.Context(), map[string]interface{}{"kb_id": "kb-1", "doc_id": "doc-1"}, "ragflow_tenant", "kb-1")
+ if err == nil {
+ t.Fatal("DeleteChunks returned nil error for a partial delete_by_query response")
+ }
+ if deleted != 2 {
+ t.Fatalf("deleted=%d, want 2", deleted)
+ }
+ })
+ }
+}
+
+// TestDeleteMetadataSurfacesPartialFailures guards the metadata side of the
+// same contract: delete_by_query answering 200 with timed_out or failures set
+// must surface as an error, otherwise RemoveDocumentKeepFile can leave orphan
+// ragflow_doc_meta_<tenant> rows (#20659).
+func TestDeleteMetadataSurfacesPartialFailures(t *testing.T) {
+ cases := map[string]string{
+ "timed_out": `{"deleted":2,"timed_out":true,"failures":[]}`,
+ "failures": `{"deleted":2,"timed_out":false,"failures":[{"index":"ragflow_doc_meta_tenant","status":500}]}`,
+ }
+ for name, response := range cases {
+ t.Run(name, func(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("X-Elastic-Product", "Elasticsearch")
+ if r.Method == http.MethodHead {
+ w.WriteHeader(http.StatusOK)
+ return
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(response))
+ }))
+ defer server.Close()
+ client, err := elasticsearch.NewClient(elasticsearch.Config{Addresses: []string{server.URL}})
+ if err != nil {
+ t.Fatalf("new elasticsearch client: %v", err)
+ }
+ engine := &Engine{client: client}
+ deleted, err := engine.DeleteMetadata(t.Context(), map[string]interface{}{"kb_id": "kb-1", "id": "doc-1"}, "tenant")
+ if err == nil {
+ t.Fatal("DeleteMetadata returned nil error for a partial delete_by_query response")
+ }
+ if deleted != 2 {
+ t.Fatalf("deleted=%d, want 2", deleted)
+ }
+ })
+ }
+}
+
func TestCloneFilterSnapshotIsolatesLaterShouldAppends(t *testing.T) {
boolMap := map[string]interface{}{
"must": []interface{}{map[string]interface{}{"term": map[string]interface{}{"kb_id": "kb-1"}}},
diff --git a/internal/engine/elasticsearch/metadata.go b/internal/engine/elasticsearch/metadata.go
index bc7f8107cf..8c9aba6ccc 100644
--- a/internal/engine/elasticsearch/metadata.go
+++ b/internal/engine/elasticsearch/metadata.go
@@ -319,6 +319,15 @@ func (e *Engine) DeleteMetadata(ctx context.Context, condition map[string]interf
deleted = int64(d)
}
+ // delete_by_query can answer 200 while only part of the work was done.
+ // Surface that so callers do not treat leftover metadata as deleted.
+ if timedOut, _ := result["timed_out"].(bool); timedOut {
+ return deleted, fmt.Errorf("elasticsearch delete metadata by query timed out after deleting %d metadata documents from %s", deleted, indexName)
+ }
+ if failures, ok := result["failures"].([]interface{}); ok && len(failures) > 0 {
+ return deleted, fmt.Errorf("elasticsearch delete metadata by query reported %d failures after deleting %d metadata documents from %s", len(failures), deleted, indexName)
+ }
+
common.Info("ElasticsearchConnection.DeleteMetadata completes", zap.String("index_name", indexName), zap.Int64("deleted_count", deleted))
return deleted, nil
}
diff --git a/internal/service/document/document_crud.go b/internal/service/document/document_crud.go
index 53a165e6f5..3d56ad97bc 100644
--- a/internal/service/document/document_crud.go
+++ b/internal/service/document/document_crud.go
@@ -293,7 +293,11 @@ func (s *DocumentService) deleteDocumentFull(ctx context.Context, docID string)
}
}
- if err := s.deleteDocEngineData(ctx, docID, kb.TenantID, doc.KbID); err != nil {
+ variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, kb.TenantID, kb.ID, docID)
+ if typeErr != nil {
+ return fmt.Errorf("resolve knowledge compile types for document %s: %w", docID, typeErr)
+ }
+ if err := s.deleteDocEngineData(ctx, docID, kb.TenantID, doc.KbID, variants, taskTypes); err != nil {
return err
}
if err = s.deleteDocRecordWithCounters(ctx, doc, kb.ID); err != nil {
@@ -313,6 +317,10 @@ func (s *DocumentService) deleteDocumentFull(ctx context.Context, docID string)
// deleting the underlying file record, its storage blob, or its file2document
// mappings. Mirrors Python DocumentService.remove_document — the caller is
// responsible for cleaning up the file2document mappings separately.
+//
+// Chunk + metadata cleanup is delegated to deleteDocEngineData so the keep-
+// file path matches deleteDocumentFull. The doc_deleted event is published
+// from inside deleteDocEngineData; no separate PublishDeleted is needed here.
func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID string) error {
doc, kb, err := s.resolveDocAndKB(ctx, docID)
if err != nil {
@@ -337,19 +345,11 @@ func (s *DocumentService) RemoveDocumentKeepFile(ctx context.Context, docID stri
}
common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: failed to delete tasks for %s: %v", docID, delErr))
}
- if err := s.deleteDocRecordWithCounters(ctx, doc, kb.ID); err != nil {
+ if err := s.deleteDocEngineData(ctx, docID, kb.TenantID, doc.KbID, variants, taskTypes); err != nil {
return err
}
- if len(variants) == 0 {
- return nil
- }
- // File replacement/deletion uses this path instead of deleteDocumentFull.
- // Publish the same deletion event so the dataset-level consumer removes the
- // deleted document's contribution in both paths.
- pubCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 15*time.Second)
- defer cancel()
- if err := knowledge_compile.PublishDeleted(pubCtx, kb.TenantID, kb.ID, docID, variants, taskTypes); err != nil {
- common.Warn(fmt.Sprintf("RemoveDocumentKeepFile: publish doc_deleted for %s failed: %v", docID, err))
+ if err := s.deleteDocRecordWithCounters(ctx, doc, kb.ID); err != nil {
+ return err
}
return nil
}
@@ -412,18 +412,22 @@ func (s *DocumentService) resolveDocAndKB(ctx context.Context, docID string) (*e
}
// deleteDocEngineData removes chunks and metadata from the document engine.
-// No-op when the engine is nil.
-func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenantID, kbID string) error {
+// No-op when the engine is nil. Callers pass the resolved compile variants and
+// task types so the file-delete hot path does not search for them twice.
+func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenantID, kbID string, variants, taskTypes []string) error {
if s.docEngine == nil {
return nil
}
indexName := fmt.Sprintf("ragflow_%s", tenantID)
- variants, taskTypes, typeErr := s.documentKnowledgeCompileTypes(ctx, tenantID, kbID, docID)
- if typeErr != nil {
- return fmt.Errorf("resolve knowledge compile types for document %s: %w", docID, typeErr)
- }
deleteCtx, cancel := context.WithTimeout(ctx, cleanupBatchTimeout)
_, delErr := s.docEngine.DeleteChunks(deleteCtx, map[string]interface{}{"doc_id": docID}, indexName, kbID)
+ if delErr != nil && deleteCtx.Err() == nil {
+ // Retry once under a fresh deadline: reusing deleteCtx would start the
+ // second attempt with whatever time the first attempt left over.
+ retryCtx, retryCancel := context.WithTimeout(ctx, cleanupBatchTimeout)
+ _, delErr = s.docEngine.DeleteChunks(retryCtx, map[string]interface{}{"doc_id": docID}, indexName, kbID)
+ retryCancel()
+ }
cancel()
if delErr != nil {
common.Warn(fmt.Sprintf("deleteDocEngineData: failed to delete chunks for %s: %v", docID, delErr))
@@ -431,7 +435,9 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenant
}
if len(variants) == 0 {
if s.metadataSvc != nil {
- _ = s.DeleteDocumentAllMetadata(ctx, docID) // logs internally
+ if err := s.DeleteDocumentAllMetadata(ctx, docID); err != nil {
+ return err
+ }
}
return nil
}
@@ -448,7 +454,9 @@ func (s *DocumentService) deleteDocEngineData(ctx context.Context, docID, tenant
common.Warn(fmt.Sprintf("deleteDocEngineData: publish doc_deleted for %s failed: %v", docID, err))
}
if s.metadataSvc != nil {
- _ = s.DeleteDocumentAllMetadata(ctx, docID) // logs internally
+ if err := s.DeleteDocumentAllMetadata(ctx, docID); err != nil {
+ return err
+ }
}
return nil
}
diff --git a/internal/service/document/document_test.go b/internal/service/document/document_test.go
index a14633ab61..6b5cbdd121 100644
--- a/internal/service/document/document_test.go
+++ b/internal/service/document/document_test.go
@@ -5440,6 +5440,246 @@ func TestFileDeleteRemovesLinkedDocument(t *testing.T) {
}
}
+// keepFileCleanupDocEngine records DeleteChunks + DeleteMetadata calls so the
+// keep-file cleanup path can assert that engine data is purged even when the
+// underlying file record + storage blob are explicitly preserved.
+type keepFileCleanupDocEngine struct {
+ fakeChatDocEngine
+ deleteChunksError error
+ deleteChunksFailOnce bool
+ deleteMetadataError error
+ deleteChunksCalls int
+ deleteChunksCondition map[string]interface{}
+ deleteChunksIndexName string
+ deleteChunksDatasetID string
+ deleteMetadataCalls int
+ deleteMetadataCond map[string]interface{}
+}
+
+func (e *keepFileCleanupDocEngine) DeleteChunks(_ context.Context, condition map[string]interface{}, indexName string, datasetID string) (int64, error) {
+ e.deleteChunksCalls++
+ e.deleteChunksCondition = condition
+ e.deleteChunksIndexName = indexName
+ e.deleteChunksDatasetID = datasetID
+ if e.deleteChunksFailOnce && e.deleteChunksCalls > 1 {
+ return 1, nil
+ }
+ return 1, e.deleteChunksError
+}
+
+func (e *keepFileCleanupDocEngine) DeleteMetadata(_ context.Context, condition map[string]interface{}, _ string) (int64, error) {
+ e.deleteMetadataCalls++
+ e.deleteMetadataCond = condition
+ return 0, e.deleteMetadataError
+}
+
+// TestRemoveDocumentKeepFileCleansUpEngineData regression-locks the
+// cycle-108 fix: RemoveDocumentKeepFile must purge chunks + metadata from
+// the engine, mirroring deleteDocumentFull. Pre-fix, the function left
+// orphans in the chunk store and ragflow_doc_meta_<tenant> table, which
+// MetadataService.GetFlattedMetaByKBs could still surface.
+func TestRemoveDocumentKeepFileCleansUpEngineData(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 30, 10)
+ insertTestDoc(t, "doc-1", "kb-1", 30, 10)
+ insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
+
+ engine := &keepFileCleanupDocEngine{}
+ svc := testDocumentService(t)
+ svc.docEngine = engine
+ svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)
+
+ if err := svc.RemoveDocumentKeepFile(t.Context(), "doc-1"); err != nil {
+ t.Fatalf("RemoveDocumentKeepFile failed: %v", err)
+ }
+
+ if engine.deleteChunksCalls != 1 {
+ t.Fatalf("DeleteChunks calls = %d, want 1", engine.deleteChunksCalls)
+ }
+ if got := engine.deleteChunksIndexName; got != "ragflow_tenant-1" {
+ t.Fatalf("DeleteChunks indexName = %q, want %q", got, "ragflow_tenant-1")
+ }
+ if got := engine.deleteChunksDatasetID; got != "kb-1" {
+ t.Fatalf("DeleteChunks datasetID = %q, want %q", got, "kb-1")
+ }
+ if got := engine.deleteChunksCondition["doc_id"]; got != "doc-1" {
+ t.Fatalf("DeleteChunks condition[doc_id] = %v, want %q", got, "doc-1")
+ }
+
+ if engine.deleteMetadataCalls != 1 {
+ t.Fatalf("DeleteMetadata calls=%d, want 1", engine.deleteMetadataCalls)
+ }
+ if engine.deleteMetadataCond["id"] != "doc-1" || engine.deleteMetadataCond["kb_id"] != "kb-1" {
+ t.Fatalf("metadata condition=%v", engine.deleteMetadataCond)
+ }
+
+ // Verify document row removed.
+ if _, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1"); err == nil {
+ t.Fatal("document row should be removed")
+ }
+
+ // Verify ingestion task removed.
+ if task, err := svc.ingestionTaskDAO.GetByDocumentID(t.Context(), db, "doc-1"); err != nil {
+ t.Fatalf("reload ingestion task: %v", err)
+ } else if task != nil {
+ t.Fatal("ingestion task should be removed")
+ }
+
+ // Verify KB counters decremented.
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
+ if err != nil {
+ t.Fatalf("reload kb: %v", err)
+ }
+ if kb.DocNum != 0 {
+ t.Fatalf("doc_num = %d, want 0", kb.DocNum)
+ }
+}
+
+// TestRemoveDocumentKeepFilePreservesFileRecord regression-locks the
+// explicit "keep file" intent: the function must NOT touch the file row,
+// the file2document mapping, or the storage blob. Pre-fix, the bug was
+// about orphan data; this test pins the inverse: the file path stays
+// untouched.
+func TestRemoveDocumentKeepFilePreservesFileRecord(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 30, 10)
+ insertTestDoc(t, "doc-1", "kb-1", 30, 10)
+ insertTestIngestionTask(t, "task-1", "user-1", "doc-1", "kb-1")
+ loc := "path/to/blob"
+ insertTestFile(t, "file-1", "dataset-folder-1", "test.pdf", &loc)
+ insertTestFile2Document(t, "f2d-1", "file-1", "doc-1")
+ store := newFakeUploadStorage()
+ if err := store.Put(t.Context(), "kb-1", loc, []byte("document")); err != nil {
+ t.Fatalf("store document blob: %v", err)
+ }
+ factory := storage.GetStorageFactory()
+ originalStorage := factory.GetStorage()
+ factory.SetStorage(store)
+ t.Cleanup(func() { factory.SetStorage(originalStorage) })
+
+ engine := &keepFileCleanupDocEngine{}
+ svc := testDocumentService(t)
+ svc.docEngine = engine
+
+ if err := svc.RemoveDocumentKeepFile(t.Context(), "doc-1"); err != nil {
+ t.Fatalf("RemoveDocumentKeepFile failed: %v", err)
+ }
+
+ // File record should still exist.
+ files, _ := dao.NewFileDAO().GetByIDs(t.Context(), db, []string{"file-1"})
+ if len(files) != 1 {
+ t.Fatalf("file record should survive, got %d", len(files))
+ }
+ // file2document mapping should still exist.
+ mappings, _ := dao.NewFile2DocumentDAO().GetByDocumentID(t.Context(), db, "doc-1")
+ if len(mappings) != 1 {
+ t.Fatalf("file2document mapping should survive, got %d", len(mappings))
+ }
+ // Storage blob should still exist.
+ if !store.ObjExist(t.Context(), "kb-1", loc) {
+ t.Fatal("storage blob should survive")
+ }
+}
+
+func TestRemoveDocumentKeepFileKeepsRowOnEngineFailure(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 30, 10)
+ insertTestDoc(t, "doc-1", "kb-1", 30, 10)
+ svc := testDocumentService(t)
+ svc.docEngine = &keepFileCleanupDocEngine{deleteChunksError: fmt.Errorf("partial delete")}
+ svc.metadataSvc = nil
+ if err := svc.RemoveDocumentKeepFile(t.Context(), "doc-1"); err == nil {
+ t.Fatal("expected engine error")
+ }
+ if _, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1"); err != nil {
+ t.Fatalf("document row must remain: %v", err)
+ }
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if kb.DocNum != 1 {
+ t.Fatalf("doc_num=%d, want 1", kb.DocNum)
+ }
+ if got := svc.docEngine.(*keepFileCleanupDocEngine).deleteChunksCalls; got != 2 {
+ t.Fatalf("failed cleanup calls=%d, want bounded retry of 2", got)
+ }
+}
+
+func TestRemoveDocumentKeepFileKeepsRowOnMetadataFailure(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 30, 10)
+ insertTestDoc(t, "doc-1", "kb-1", 30, 10)
+ engine := &keepFileCleanupDocEngine{deleteMetadataError: fmt.Errorf("metadata unavailable")}
+ svc := testDocumentService(t)
+ svc.docEngine = engine
+ svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)
+ if err := svc.RemoveDocumentKeepFile(t.Context(), "doc-1"); err == nil {
+ t.Fatal("expected metadata error")
+ }
+ if _, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1"); err != nil {
+ t.Fatalf("document row must remain: %v", err)
+ }
+ if engine.deleteMetadataCalls != 1 {
+ t.Fatalf("metadata calls=%d", engine.deleteMetadataCalls)
+ }
+}
+
+// TestDeleteDocumentFullKeepsDocumentWhenMetadataDeletionFails pins the
+// shared-helper contract on the full path too: a metadata cleanup failure
+// surfaces before the document row is removed, so a retry can complete the
+// delete instead of leaving orphan ragflow_doc_meta_<tenant> records.
+func TestDeleteDocumentFullKeepsDocumentWhenMetadataDeletionFails(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 30, 10)
+ insertTestDoc(t, "doc-1", "kb-1", 30, 10)
+ engine := &keepFileCleanupDocEngine{deleteMetadataError: fmt.Errorf("metadata unavailable")}
+ svc := testDocumentService(t)
+ svc.docEngine = engine
+ svc.metadataSvc = service.NewMetadataServiceForTest(dao.NewKnowledgebaseDAO(), engine)
+ if err := svc.deleteDocumentFull(t.Context(), "doc-1"); err == nil {
+ t.Fatal("expected metadata error")
+ }
+ if _, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1"); err != nil {
+ t.Fatalf("document row must remain after metadata failure: %v", err)
+ }
+ kb, err := dao.NewKnowledgebaseDAO().GetByID(t.Context(), db, "kb-1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if kb.DocNum != 1 {
+ t.Fatalf("doc_num=%d, want 1", kb.DocNum)
+ }
+ if engine.deleteMetadataCalls != 1 {
+ t.Fatalf("metadata calls=%d, want 1", engine.deleteMetadataCalls)
+ }
+}
+
+func TestRemoveDocumentKeepFileRetriesEngineFailure(t *testing.T) {
+ db := setupServiceTestDB(t)
+ pushServiceDB(t, db)
+ insertTestKB(t, "kb-1", "tenant-1", 1, 30, 10)
+ insertTestDoc(t, "doc-1", "kb-1", 30, 10)
+ engine := &keepFileCleanupDocEngine{deleteChunksError: fmt.Errorf("partial delete"), deleteChunksFailOnce: true}
+ svc := testDocumentService(t)
+ svc.docEngine = engine
+ svc.metadataSvc = nil
+ if err := svc.RemoveDocumentKeepFile(t.Context(), "doc-1"); err != nil {
+ t.Fatalf("retry: %v", err)
+ }
+ if engine.deleteChunksCalls != 2 {
+ t.Fatalf("calls=%d, want 2", engine.deleteChunksCalls)
+ }
+ if _, err := dao.NewDocumentDAO().GetByID(t.Context(), db, "doc-1"); err == nil {
+ t.Fatal("document row should be removed after cleanup succeeds")
+ }
+}
+
func TestDownloadEmptyDocumentAndMissingBlob(t *testing.T) {
db := setupServiceTestDB(t)
pushServiceDB(t, db)