Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(knowledge): restore downloads after converting datasets to pipelines (#45428)

FFXNc3780668b85e
diff --git a/api/services/knowledge/dataset_service.py b/api/services/knowledge/dataset_service.py
index 4e30880da7a..9622fed6795 100644
--- a/api/services/knowledge/dataset_service.py
+++ b/api/services/knowledge/dataset_service.py
@@ -24,6 +24,7 @@ from core.model_manager import ModelManager
from core.rag.index_processor.constant.built_in_field import BuiltInField
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from core.rag.retrieval.retrieval_methods import RetrievalMethod
+from core.workflow.file_reference import resolve_file_record_id
from enums import CloudPlan, DeploymentEdition
from events.dataset_event import dataset_was_deleted
from events.document_event import document_was_deleted
@@ -1665,8 +1666,10 @@ class DocumentService:
@staticmethod
def get_document_download_url(document: Document, session: Session) -> str:
- """
- Return a signed download URL for an upload-file document.
+ """Return a signed URL for a file-backed document in the caller's tenant.
+
+ Requires a legacy upload or pipeline local-file reference to an existing tenant-owned
+ UploadFile. On success the URL downloads that file; invalid or missing links raise 404.
"""
upload_file = DocumentService._get_upload_file_for_upload_file_document(document, session)
return file_helpers.get_signed_file_url(upload_file_id=upload_file.id, as_attachment=True)
@@ -1762,14 +1765,25 @@ class DocumentService:
invalid_source_message: str,
missing_file_message: str,
) -> str:
- """
- Normalize and validate `Document -> UploadFile` linkage for download flows.
- """
- if document.data_source_type != DataSourceType.UPLOAD_FILE:
- raise NotFound(invalid_source_message)
+ """Resolve a file-backed document's UploadFile id without trusting its source metadata.
+ Legacy uploads use ``upload_file_id``; pipeline local files use ``real_file_id``
+ after conversion or ``related_id``/``reference`` when created in a pipeline.
+ Other source types and absent references raise 404; callers must still load the
+ UploadFile within the document's tenant before using the resolved id.
+ """
data_source_info: dict[str, Any] = document.data_source_info_dict or {}
- upload_file_id: str | None = data_source_info.get("upload_file_id")
+ if document.data_source_type == DataSourceType.UPLOAD_FILE:
+ upload_file_id = data_source_info.get("upload_file_id")
+ elif document.data_source_type == DataSourceType.LOCAL_FILE:
+ file_reference = (
+ data_source_info.get("real_file_id")
+ or data_source_info.get("related_id")
+ or data_source_info.get("reference")
+ )
+ upload_file_id = resolve_file_record_id(file_reference) if isinstance(file_reference, str) else None
+ else:
+ raise NotFound(invalid_source_message)
if not upload_file_id:
raise NotFound(missing_file_message)
@@ -1777,9 +1791,7 @@ class DocumentService:
@staticmethod
def _get_upload_file_for_upload_file_document(document: Document, session: Session) -> UploadFile:
- """
- Load the `UploadFile` row for an upload-file document.
- """
+ """Load a file-backed document's UploadFile within its tenant or raise 404."""
upload_file_id = DocumentService._get_upload_file_id_for_upload_file_document(
document,
invalid_source_message="Document does not have an uploaded file to download.",
@@ -1799,8 +1811,10 @@ class DocumentService:
tenant_id: str,
session: Session,
) -> dict[str, UploadFile]:
- """
- Batch load upload files keyed by document id for ZIP downloads.
+ """Load tenant-owned UploadFiles for file-backed documents keyed by document id.
+
+ Requires every requested document and file to exist in the given dataset and tenant.
+ Returns a complete mapping on success; missing or unsupported sources raise 404.
"""
document_id_list: list[str] = list(document_ids)
diff --git a/api/tests/unit_tests/services/knowledge/test_dataset_service.py b/api/tests/unit_tests/services/knowledge/test_dataset_service.py
index 95cf2a6d3ab..214dbc4db4a 100644
--- a/api/tests/unit_tests/services/knowledge/test_dataset_service.py
+++ b/api/tests/unit_tests/services/knowledge/test_dataset_service.py
@@ -27,6 +27,7 @@ from core.rag.index_processor.constant.index_type import (
)
from core.rag.index_processor.constant.index_type import IndexTechniqueType
from core.rag.retrieval.dataset_retrieval import DatasetRetrieval
+from core.workflow.file_reference import build_file_reference
from enums import DeploymentEdition
from graphon.model_runtime.entities.model_entities import ModelFeature, ModelType
from models import Account as _dataset_service_dataset_Account
@@ -1919,6 +1920,76 @@ class _RetryFlagStore:
return _RetryFlagLock(self, key)
+class TestDocumentServiceDownloads:
+ @pytest.mark.parametrize("sqlite_session", [(_dataset_service_document_Document, UploadFile)], indirect=True)
+ @pytest.mark.parametrize(
+ ("source_type", "source_info"),
+ [
+ (DataSourceType.UPLOAD_FILE, {"upload_file_id": "file-1"}),
+ (DataSourceType.LOCAL_FILE, {"real_file_id": "file-1"}),
+ (DataSourceType.LOCAL_FILE, {"related_id": "file-1"}),
+ (DataSourceType.LOCAL_FILE, {"reference": build_file_reference(record_id="file-1")}),
+ ],
+ )
+ def test_single_and_zip_download_resolve_tenant_file(
+ self,
+ sqlite_session: Session,
+ source_type: DataSourceType,
+ source_info: dict[str, str],
+ ) -> None:
+ document = make_document(data_source_type=source_type, data_source_info=json.dumps(source_info))
+ upload_file = make_upload_file(file_id="file-1")
+ sqlite_session.add_all([document, upload_file])
+ sqlite_session.flush()
+
+ with _dataset_service_document_patch(
+ "services.knowledge.dataset_service.file_helpers.get_signed_file_url", return_value="signed-url"
+ ) as get_signed_url:
+ assert (
+ _dataset_service_document_DocumentService.get_document_download_url(document, sqlite_session)
+ == "signed-url"
+ )
+ get_signed_url.assert_called_once_with(upload_file_id=upload_file.id, as_attachment=True)
+
+ files = _dataset_service_document_DocumentService._get_upload_files_by_document_id_for_zip_download(
+ dataset_id=document.dataset_id,
+ document_ids=[document.id],
+ tenant_id=document.tenant_id,
+ session=sqlite_session,
+ )
+ assert files == {document.id: upload_file}
+
+ @pytest.mark.parametrize("sqlite_session", [(_dataset_service_document_Document, UploadFile)], indirect=True)
+ @pytest.mark.parametrize(
+ ("source_type", "source_info", "file_tenant_id"),
+ [
+ (DataSourceType.LOCAL_FILE, {}, "tenant-1"),
+ (DataSourceType.LOCAL_FILE, {"real_file_id": "file-1"}, "other-tenant"),
+ (DataSourceType.NOTION_IMPORT, {"real_file_id": "file-1"}, "tenant-1"),
+ ],
+ )
+ def test_download_rejects_missing_foreign_or_non_file_source(
+ self,
+ sqlite_session: Session,
+ source_type: DataSourceType,
+ source_info: dict[str, str],
+ file_tenant_id: str,
+ ) -> None:
+ document = make_document(data_source_type=source_type, data_source_info=json.dumps(source_info))
+ sqlite_session.add_all([document, make_upload_file(file_id="file-1", tenant_id=file_tenant_id)])
+ sqlite_session.flush()
+
+ with pytest.raises(NotFound):
+ _dataset_service_document_DocumentService.get_document_download_url(document, sqlite_session)
+ with pytest.raises(NotFound):
+ _dataset_service_document_DocumentService._get_upload_files_by_document_id_for_zip_download(
+ dataset_id=document.dataset_id,
+ document_ids=[document.id],
+ tenant_id=document.tenant_id,
+ session=sqlite_session,
+ )
+
+
class TestDatasetServiceDocumentCases:
class TestDocumentServiceDisplayStatus:
"""Unit tests for DocumentService display-status helpers."""
diff --git a/web/app/components/datasets/documents/__tests__/document-source.spec.ts b/web/app/components/datasets/documents/__tests__/document-source.spec.ts
index e01f78eb846..fa478846ffe 100644
--- a/web/app/components/datasets/documents/__tests__/document-source.spec.ts
+++ b/web/app/components/datasets/documents/__tests__/document-source.spec.ts
@@ -4,9 +4,21 @@ import {
getNotionPageIcon,
getOnlineDriveFileName,
getUploadFileExtension,
+ isDownloadableFileDocument,
} from '../document-source'
describe('document source info', () => {
+ it('recognizes legacy and pipeline documents with downloadable files', () => {
+ expect(isDownloadableFileDocument('upload_file', null)).toBe(true)
+ expect(isDownloadableFileDocument('local_file', { real_file_id: 'file-1' })).toBe(true)
+ expect(isDownloadableFileDocument('local_file', { related_id: 'file-2' })).toBe(true)
+ expect(isDownloadableFileDocument('local_file', { reference: 'dify-file-ref:encoded' })).toBe(
+ true,
+ )
+ expect(isDownloadableFileDocument('local_file', { file_path: '/tmp/document.pdf' })).toBe(false)
+ expect(isDownloadableFileDocument('online_document', { related_id: 'file-2' })).toBe(false)
+ })
+
it('should read the uploaded file extension of legacy file documents', () => {
expect(getUploadFileExtension({ upload_file: { extension: 'pdf' } })).toBe('pdf')
expect(getUploadFileExtension({ upload_file: 'broken' })).toBeUndefined()
diff --git a/web/app/components/datasets/documents/components/__tests__/operations.spec.tsx b/web/app/components/datasets/documents/components/__tests__/operations.spec.tsx
index 04c5cf8fe05..3cc9ab140ef 100644
--- a/web/app/components/datasets/documents/components/__tests__/operations.spec.tsx
+++ b/web/app/components/datasets/documents/components/__tests__/operations.spec.tsx
@@ -396,6 +396,41 @@ describe('Operations', () => {
})
})
+ it('should keep a converted file downloadable after its source data refreshes', async () => {
+ const user = userEvent.setup()
+ const { rerender } = render(<Operations {...defaultProps} />)
+ await openPopover(user)
+ expect(
+ screen.getByRole('menuitem', { name: 'datasetDocuments.list.action.download' }),
+ ).toBeInTheDocument()
+
+ rerender(
+ <>
+ <Operations
+ {...defaultProps}
+ detail={{
+ ...defaultDetail,
+ data_source_type: 'local_file',
+ data_source_info: { real_file_id: 'file-1' },
+ }}
+ />
+ <AppToaster />
+ </>,
+ )
+ await user.click(
+ screen.getByRole('menuitem', { name: 'datasetDocuments.list.action.download' }),
+ )
+
+ await waitFor(() => {
+ expect(mockDownload).toHaveBeenCalledWith({
+ params: { dataset_id: 'dataset-1', document_id: 'doc-1' },
+ })
+ expect(downloads).toEqual([
+ { url: 'https://example.com/download', fileName: 'Test Document' },
+ ])
+ })
+ })
+
it('should show settings for archived non-file data source', async () => {
const user = userEvent.setup()
render(
diff --git a/web/app/components/datasets/documents/components/document-list/components/document-table-row.tsx b/web/app/components/datasets/documents/components/document-list/components/document-table-row.tsx
index 024ab352464..93c73262ebf 100644
--- a/web/app/components/datasets/documents/components/document-list/components/document-table-row.tsx
+++ b/web/app/components/datasets/documents/components/document-list/components/document-table-row.tsx
@@ -195,6 +195,7 @@ const DocumentTableRow = React.memo(
'archived',
'id',
'data_source_type',
+ 'data_source_info',
'doc_form',
'display_status',
])}
diff --git a/web/app/components/datasets/documents/components/document-list/hooks/__tests__/use-document-selection.spec.ts b/web/app/components/datasets/documents/components/document-list/hooks/__tests__/use-document-selection.spec.ts
index b2f4588ac5a..0c22183b356 100644
--- a/web/app/components/datasets/documents/components/document-list/hooks/__tests__/use-document-selection.spec.ts
+++ b/web/app/components/datasets/documents/components/document-list/hooks/__tests__/use-document-selection.spec.ts
@@ -64,23 +64,33 @@ describe('useDocumentSelection', () => {
})
describe('downloadableSelectedIds', () => {
- it('should return only FILE type documents from selection', () => {
+ it('should include uploaded and pipeline files with download references', () => {
const docs = [
createMockDocument({ id: 'doc1', data_source_type: DataSourceType.FILE }),
createMockDocument({ id: 'doc2', data_source_type: DataSourceType.NOTION }),
- createMockDocument({ id: 'doc3', data_source_type: DataSourceType.FILE }),
+ createMockDocument({
+ id: 'doc3',
+ data_source_type: 'local_file',
+ data_source_info: { real_file_id: 'file-3' },
+ }),
+ createMockDocument({
+ id: 'doc4',
+ data_source_type: 'local_file',
+ data_source_info: { related_id: 'file-4' },
+ }),
+ createMockDocument({ id: 'doc5', data_source_type: 'local_file', data_source_info: {} }),
]
const onSelectedIdChange = vi.fn()
const { result } = renderHook(() =>
useDocumentSelection({
documents: docs,
- selectedIds: ['doc1', 'doc2', 'doc3'],
+ selectedIds: ['doc1', 'doc2', 'doc3', 'doc4', 'doc5'],
onSelectedIdChange,
}),
)
- expect(result.current.downloadableSelectedIds).toEqual(['doc1', 'doc3'])
+ expect(result.current.downloadableSelectedIds).toEqual(['doc1', 'doc3', 'doc4'])
})
it('should return empty array when no FILE documents selected', () => {
diff --git a/web/app/components/datasets/documents/components/document-list/hooks/use-document-selection.ts b/web/app/components/datasets/documents/components/document-list/hooks/use-document-selection.ts
index 4eaa638af28..c96813bac07 100644
--- a/web/app/components/datasets/documents/components/document-list/hooks/use-document-selection.ts
+++ b/web/app/components/datasets/documents/components/document-list/hooks/use-document-selection.ts
@@ -1,6 +1,6 @@
import type { DocumentWithSegmentsResponse } from '@dify/contracts/api/console/datasets/types.gen'
import { useCallback, useMemo } from 'react'
-import { DataSourceType } from '@/models/datasets'
+import { isDownloadableFileDocument } from '../../../document-source'
type LocalDoc = DocumentWithSegmentsResponse & { percent?: number }
@@ -22,7 +22,11 @@ export const useDocumentSelection = ({
const downloadableSelectedIds = useMemo(() => {
const selectedSet = new Set(selectedIds)
return documents
- .filter((doc) => selectedSet.has(doc.id) && doc.data_source_type === DataSourceType.FILE)
+ .filter(
+ (doc) =>
+ selectedSet.has(doc.id) &&
+ isDownloadableFileDocument(doc.data_source_type, doc.data_source_info),
+ )
.map((doc) => doc.id)
}, [documents, selectedIds])
diff --git a/web/app/components/datasets/documents/components/operations.tsx b/web/app/components/datasets/documents/components/operations.tsx
index d479990db79..d49f33abc17 100644
--- a/web/app/components/datasets/documents/components/operations.tsx
+++ b/web/app/components/datasets/documents/components/operations.tsx
@@ -11,6 +11,7 @@ import { noop } from 'es-toolkit/function'
import * as React from 'react'
import { useCallback, useState } from 'react'
import { useTranslation } from 'react-i18next'
+import { isDownloadableFileDocument } from '@/app/components/datasets/documents/document-source'
import { toast } from '@/app/notifications'
import { systemFeaturesQueryOptions } from '@/features/system-features/client'
import { DataSourceType, DocumentActionType } from '@/models/datasets'
@@ -29,6 +30,7 @@ type OperationsProps = {
enabled?: boolean | null
archived?: boolean | null
data_source_type?: string | null
+ data_source_info?: Record<string, unknown> | null
doc_form?: string | null
display_status?: string | null
}
@@ -60,7 +62,7 @@ const Operations = ({
select: ({ deployment_edition }) => deployment_edition,
})
const isNonCloudEdition = deploymentEdition === 'COMMUNITY' || deploymentEdition === 'ENTERPRISE'
- const { id, name, data_source_type, display_status } = detail
+ const { id, name, data_source_type, data_source_info, display_status } = detail
const enabled = detail.enabled ?? false
const archived = detail.archived ?? false
const { t } = useTranslation(['common', 'datasetDocuments'])
@@ -91,7 +93,8 @@ const Operations = ({
const canShowRenameAction = canEdit && !archived
const canShowSummaryAction = canEdit && !archived && isNonCloudEdition
const canShowSettingsAction = canViewSettings
- const canShowDownloadAction = canDownload && data_source_type === DataSourceType.FILE
+ const canShowDownloadAction =
+ canDownload && isDownloadableFileDocument(data_source_type, data_source_info)
const canShowSyncAction =
canEdit &&
!archived &&
diff --git a/web/app/components/datasets/documents/detail/index.tsx b/web/app/components/datasets/documents/detail/index.tsx
index cadc5a43913..ec2a7c3de0c 100644
--- a/web/app/components/datasets/documents/detail/index.tsx
+++ b/web/app/components/datasets/documents/detail/index.tsx
@@ -282,6 +282,7 @@ const DocumentDetail: FC<DocumentDetailProps> = ({ datasetId, documentId }) => {
archived: documentDetail?.archived || false,
id: documentId,
data_source_type: documentDetail?.data_source_type || '',
+ data_source_info: documentDetail?.data_source_info,
doc_form: documentDetail?.doc_form || '',
}),
[
@@ -290,6 +291,7 @@ const DocumentDetail: FC<DocumentDetailProps> = ({ datasetId, documentId }) => {
documentDetail?.archived,
documentId,
documentDetail?.data_source_type,
+ documentDetail?.data_source_info,
documentDetail?.doc_form,
],
)
diff --git a/web/app/components/datasets/documents/document-source.ts b/web/app/components/datasets/documents/document-source.ts
index 575b00b4330..6bc07d5cbad 100644
--- a/web/app/components/datasets/documents/document-source.ts
+++ b/web/app/components/datasets/documents/document-source.ts
@@ -1,4 +1,6 @@
import type { DataSourceNotionPage } from '@/models/common'
+import { DataSourceType } from '@/models/datasets'
+import { DatasourceType } from '@/models/pipeline'
// data_source_info is written by the data source that imported the document, so its keys are
// read defensively for each source type.
@@ -12,6 +14,21 @@ const getString = (record: Record<string, unknown> | undefined, key: string) =>
return typeof value === 'string' ? value : undefined
}
+/** Whether a document source has a file reference supported by the download API. */
+export const isDownloadableFileDocument = (
+ sourceType: string | null | undefined,
+ info: DocumentSourceInfo,
+) => {
+ if (sourceType === DataSourceType.FILE) return true
+ if (sourceType !== DatasourceType.localFile) return false
+
+ return Boolean(
+ getString(info ?? undefined, 'real_file_id') ||
+ getString(info ?? undefined, 'related_id') ||
+ getString(info ?? undefined, 'reference'),
+ )
+}
+
const isNotionPageIcon = (
value: unknown,
): value is NonNullable<DataSourceNotionPage['page_icon']> =>