Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
fix(knowledge): restore downloads after converting datasets to pipelines (#45428)
FFXNc3780668b85e
diff --git a/api/services/knowledge/dataset_service.py b/api/services/knowledge/dataset_service.pyindex 4e30880da7a..9622fed6795 100644--- a/api/services/knowledge/dataset_service.py+++ b/api/services/knowledge/dataset_service.py@@ -24,6 +24,7 @@ from core.model_manager import ModelManagerfrom core.rag.index_processor.constant.built_in_field import BuiltInFieldfrom core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueTypefrom core.rag.retrieval.retrieval_methods import RetrievalMethod+from core.workflow.file_reference import resolve_file_record_idfrom enums import CloudPlan, DeploymentEditionfrom events.dataset_event import dataset_was_deletedfrom events.document_event import document_was_deleted@@ -1665,8 +1666,10 @@ class DocumentService:@staticmethoddef get_document_download_url(document: Document, session: Session) -> str:- """- Return a signed download URL for an upload-file document.+ """Return a signed URL for a file-backed document in the caller's tenant.++ Requires a legacy upload or pipeline local-file reference to an existing tenant-owned+ UploadFile. On success the URL downloads that file; invalid or missing links raise 404."""upload_file = DocumentService._get_upload_file_for_upload_file_document(document, session)return file_helpers.get_signed_file_url(upload_file_id=upload_file.id, as_attachment=True)@@ -1762,14 +1765,25 @@ class DocumentService:invalid_source_message: str,missing_file_message: str,) -> str:- """- Normalize and validate `Document -> UploadFile` linkage for download flows.- """- if document.data_source_type != DataSourceType.UPLOAD_FILE:- raise NotFound(invalid_source_message)+ """Resolve a file-backed document's UploadFile id without trusting its source metadata.+ Legacy uploads use ``upload_file_id``; pipeline local files use ``real_file_id``+ after conversion or ``related_id``/``reference`` when created in a pipeline.+ Other source types and absent references raise 404; callers must still load the+ UploadFile within the document's tenant before using the resolved id.+ """data_source_info: dict[str, Any] = document.data_source_info_dict or {}- upload_file_id: str | None = data_source_info.get("upload_file_id")+ if document.data_source_type == DataSourceType.UPLOAD_FILE:+ upload_file_id = data_source_info.get("upload_file_id")+ elif document.data_source_type == DataSourceType.LOCAL_FILE:+ file_reference = (+ data_source_info.get("real_file_id")+ or data_source_info.get("related_id")+ or data_source_info.get("reference")+ )+ upload_file_id = resolve_file_record_id(file_reference) if isinstance(file_reference, str) else None+ else:+ raise NotFound(invalid_source_message)if not upload_file_id:raise NotFound(missing_file_message)@@ -1777,9 +1791,7 @@ class DocumentService:@staticmethoddef _get_upload_file_for_upload_file_document(document: Document, session: Session) -> UploadFile:- """- Load the `UploadFile` row for an upload-file document.- """+ """Load a file-backed document's UploadFile within its tenant or raise 404."""upload_file_id = DocumentService._get_upload_file_id_for_upload_file_document(document,invalid_source_message="Document does not have an uploaded file to download.",@@ -1799,8 +1811,10 @@ class DocumentService:tenant_id: str,session: Session,) -> dict[str, UploadFile]:- """- Batch load upload files keyed by document id for ZIP downloads.+ """Load tenant-owned UploadFiles for file-backed documents keyed by document id.++ Requires every requested document and file to exist in the given dataset and tenant.+ Returns a complete mapping on success; missing or unsupported sources raise 404."""document_id_list: list[str] = list(document_ids)diff --git a/api/tests/unit_tests/services/knowledge/test_dataset_service.py b/api/tests/unit_tests/services/knowledge/test_dataset_service.pyindex 95cf2a6d3ab..214dbc4db4a 100644--- a/api/tests/unit_tests/services/knowledge/test_dataset_service.py+++ b/api/tests/unit_tests/services/knowledge/test_dataset_service.py@@ -27,6 +27,7 @@ from core.rag.index_processor.constant.index_type import ()from core.rag.index_processor.constant.index_type import IndexTechniqueTypefrom core.rag.retrieval.dataset_retrieval import DatasetRetrieval+from core.workflow.file_reference import build_file_referencefrom enums import DeploymentEditionfrom graphon.model_runtime.entities.model_entities import ModelFeature, ModelTypefrom models import Account as _dataset_service_dataset_Account@@ -1919,6 +1920,76 @@ class _RetryFlagStore:return _RetryFlagLock(self, key)+class TestDocumentServiceDownloads:+ @pytest.mark.parametrize("sqlite_session", [(_dataset_service_document_Document, UploadFile)], indirect=True)+ @pytest.mark.parametrize(+ ("source_type", "source_info"),+ [+ (DataSourceType.UPLOAD_FILE, {"upload_file_id": "file-1"}),+ (DataSourceType.LOCAL_FILE, {"real_file_id": "file-1"}),+ (DataSourceType.LOCAL_FILE, {"related_id": "file-1"}),+ (DataSourceType.LOCAL_FILE, {"reference": build_file_reference(record_id="file-1")}),+ ],+ )+ def test_single_and_zip_download_resolve_tenant_file(+ self,+ sqlite_session: Session,+ source_type: DataSourceType,+ source_info: dict[str, str],+ ) -> None:+ document = make_document(data_source_type=source_type, data_source_info=json.dumps(source_info))+ upload_file = make_upload_file(file_id="file-1")+ sqlite_session.add_all([document, upload_file])+ sqlite_session.flush()++ with _dataset_service_document_patch(+ "services.knowledge.dataset_service.file_helpers.get_signed_file_url", return_value="signed-url"+ ) as get_signed_url:+ assert (+ _dataset_service_document_DocumentService.get_document_download_url(document, sqlite_session)+ == "signed-url"+ )+ get_signed_url.assert_called_once_with(upload_file_id=upload_file.id, as_attachment=True)++ files = _dataset_service_document_DocumentService._get_upload_files_by_document_id_for_zip_download(+ dataset_id=document.dataset_id,+ document_ids=[document.id],+ tenant_id=document.tenant_id,+ session=sqlite_session,+ )+ assert files == {document.id: upload_file}++ @pytest.mark.parametrize("sqlite_session", [(_dataset_service_document_Document, UploadFile)], indirect=True)+ @pytest.mark.parametrize(+ ("source_type", "source_info", "file_tenant_id"),+ [+ (DataSourceType.LOCAL_FILE, {}, "tenant-1"),+ (DataSourceType.LOCAL_FILE, {"real_file_id": "file-1"}, "other-tenant"),+ (DataSourceType.NOTION_IMPORT, {"real_file_id": "file-1"}, "tenant-1"),+ ],+ )+ def test_download_rejects_missing_foreign_or_non_file_source(+ self,+ sqlite_session: Session,+ source_type: DataSourceType,+ source_info: dict[str, str],+ file_tenant_id: str,+ ) -> None:+ document = make_document(data_source_type=source_type, data_source_info=json.dumps(source_info))+ sqlite_session.add_all([document, make_upload_file(file_id="file-1", tenant_id=file_tenant_id)])+ sqlite_session.flush()++ with pytest.raises(NotFound):+ _dataset_service_document_DocumentService.get_document_download_url(document, sqlite_session)+ with pytest.raises(NotFound):+ _dataset_service_document_DocumentService._get_upload_files_by_document_id_for_zip_download(+ dataset_id=document.dataset_id,+ document_ids=[document.id],+ tenant_id=document.tenant_id,+ session=sqlite_session,+ )++class TestDatasetServiceDocumentCases:class TestDocumentServiceDisplayStatus:"""Unit tests for DocumentService display-status helpers."""diff --git a/web/app/components/datasets/documents/__tests__/document-source.spec.ts b/web/app/components/datasets/documents/__tests__/document-source.spec.tsindex e01f78eb846..fa478846ffe 100644--- a/web/app/components/datasets/documents/__tests__/document-source.spec.ts+++ b/web/app/components/datasets/documents/__tests__/document-source.spec.ts@@ -4,9 +4,21 @@ import {getNotionPageIcon,getOnlineDriveFileName,getUploadFileExtension,+ isDownloadableFileDocument,} from '../document-source'describe('document source info', () => {+ it('recognizes legacy and pipeline documents with downloadable files', () => {+ expect(isDownloadableFileDocument('upload_file', null)).toBe(true)+ expect(isDownloadableFileDocument('local_file', { real_file_id: 'file-1' })).toBe(true)+ expect(isDownloadableFileDocument('local_file', { related_id: 'file-2' })).toBe(true)+ expect(isDownloadableFileDocument('local_file', { reference: 'dify-file-ref:encoded' })).toBe(+ true,+ )+ expect(isDownloadableFileDocument('local_file', { file_path: '/tmp/document.pdf' })).toBe(false)+ expect(isDownloadableFileDocument('online_document', { related_id: 'file-2' })).toBe(false)+ })+it('should read the uploaded file extension of legacy file documents', () => {expect(getUploadFileExtension({ upload_file: { extension: 'pdf' } })).toBe('pdf')expect(getUploadFileExtension({ upload_file: 'broken' })).toBeUndefined()diff --git a/web/app/components/datasets/documents/components/__tests__/operations.spec.tsx b/web/app/components/datasets/documents/components/__tests__/operations.spec.tsxindex 04c5cf8fe05..3cc9ab140ef 100644--- a/web/app/components/datasets/documents/components/__tests__/operations.spec.tsx+++ b/web/app/components/datasets/documents/components/__tests__/operations.spec.tsx@@ -396,6 +396,41 @@ describe('Operations', () => {})})+ it('should keep a converted file downloadable after its source data refreshes', async () => {+ const user = userEvent.setup()+ const { rerender } = render(<Operations {...defaultProps} />)+ await openPopover(user)+ expect(+ screen.getByRole('menuitem', { name: 'datasetDocuments.list.action.download' }),+ ).toBeInTheDocument()++ rerender(+ <>+ <Operations+ {...defaultProps}+ detail={{+ ...defaultDetail,+ data_source_type: 'local_file',+ data_source_info: { real_file_id: 'file-1' },+ }}+ />+ <AppToaster />+ </>,+ )+ await user.click(+ screen.getByRole('menuitem', { name: 'datasetDocuments.list.action.download' }),+ )++ await waitFor(() => {+ expect(mockDownload).toHaveBeenCalledWith({+ params: { dataset_id: 'dataset-1', document_id: 'doc-1' },+ })+ expect(downloads).toEqual([+ { url: 'https://example.com/download', fileName: 'Test Document' },+ ])+ })+ })+it('should show settings for archived non-file data source', async () => {const user = userEvent.setup()render(diff --git a/web/app/components/datasets/documents/components/document-list/components/document-table-row.tsx b/web/app/components/datasets/documents/components/document-list/components/document-table-row.tsxindex 024ab352464..93c73262ebf 100644--- a/web/app/components/datasets/documents/components/document-list/components/document-table-row.tsx+++ b/web/app/components/datasets/documents/components/document-list/components/document-table-row.tsx@@ -195,6 +195,7 @@ const DocumentTableRow = React.memo('archived','id','data_source_type',+ 'data_source_info','doc_form','display_status',])}diff --git a/web/app/components/datasets/documents/components/document-list/hooks/__tests__/use-document-selection.spec.ts b/web/app/components/datasets/documents/components/document-list/hooks/__tests__/use-document-selection.spec.tsindex b2f4588ac5a..0c22183b356 100644--- a/web/app/components/datasets/documents/components/document-list/hooks/__tests__/use-document-selection.spec.ts+++ b/web/app/components/datasets/documents/components/document-list/hooks/__tests__/use-document-selection.spec.ts@@ -64,23 +64,33 @@ describe('useDocumentSelection', () => {})describe('downloadableSelectedIds', () => {- it('should return only FILE type documents from selection', () => {+ it('should include uploaded and pipeline files with download references', () => {const docs = [createMockDocument({ id: 'doc1', data_source_type: DataSourceType.FILE }),createMockDocument({ id: 'doc2', data_source_type: DataSourceType.NOTION }),- createMockDocument({ id: 'doc3', data_source_type: DataSourceType.FILE }),+ createMockDocument({+ id: 'doc3',+ data_source_type: 'local_file',+ data_source_info: { real_file_id: 'file-3' },+ }),+ createMockDocument({+ id: 'doc4',+ data_source_type: 'local_file',+ data_source_info: { related_id: 'file-4' },+ }),+ createMockDocument({ id: 'doc5', data_source_type: 'local_file', data_source_info: {} }),]const onSelectedIdChange = vi.fn()const { result } = renderHook(() =>useDocumentSelection({documents: docs,- selectedIds: ['doc1', 'doc2', 'doc3'],+ selectedIds: ['doc1', 'doc2', 'doc3', 'doc4', 'doc5'],onSelectedIdChange,}),)- expect(result.current.downloadableSelectedIds).toEqual(['doc1', 'doc3'])+ expect(result.current.downloadableSelectedIds).toEqual(['doc1', 'doc3', 'doc4'])})it('should return empty array when no FILE documents selected', () => {diff --git a/web/app/components/datasets/documents/components/document-list/hooks/use-document-selection.ts b/web/app/components/datasets/documents/components/document-list/hooks/use-document-selection.tsindex 4eaa638af28..c96813bac07 100644--- a/web/app/components/datasets/documents/components/document-list/hooks/use-document-selection.ts+++ b/web/app/components/datasets/documents/components/document-list/hooks/use-document-selection.ts@@ -1,6 +1,6 @@import type { DocumentWithSegmentsResponse } from '@dify/contracts/api/console/datasets/types.gen'import { useCallback, useMemo } from 'react'-import { DataSourceType } from '@/models/datasets'+import { isDownloadableFileDocument } from '../../../document-source'type LocalDoc = DocumentWithSegmentsResponse & { percent?: number }@@ -22,7 +22,11 @@ export const useDocumentSelection = ({const downloadableSelectedIds = useMemo(() => {const selectedSet = new Set(selectedIds)return documents- .filter((doc) => selectedSet.has(doc.id) && doc.data_source_type === DataSourceType.FILE)+ .filter(+ (doc) =>+ selectedSet.has(doc.id) &&+ isDownloadableFileDocument(doc.data_source_type, doc.data_source_info),+ ).map((doc) => doc.id)}, [documents, selectedIds])diff --git a/web/app/components/datasets/documents/components/operations.tsx b/web/app/components/datasets/documents/components/operations.tsxindex d479990db79..d49f33abc17 100644--- a/web/app/components/datasets/documents/components/operations.tsx+++ b/web/app/components/datasets/documents/components/operations.tsx@@ -11,6 +11,7 @@ import { noop } from 'es-toolkit/function'import * as React from 'react'import { useCallback, useState } from 'react'import { useTranslation } from 'react-i18next'+import { isDownloadableFileDocument } from '@/app/components/datasets/documents/document-source'import { toast } from '@/app/notifications'import { systemFeaturesQueryOptions } from '@/features/system-features/client'import { DataSourceType, DocumentActionType } from '@/models/datasets'@@ -29,6 +30,7 @@ type OperationsProps = {enabled?: boolean | nullarchived?: boolean | nulldata_source_type?: string | null+ data_source_info?: Record<string, unknown> | nulldoc_form?: string | nulldisplay_status?: string | null}@@ -60,7 +62,7 @@ const Operations = ({select: ({ deployment_edition }) => deployment_edition,})const isNonCloudEdition = deploymentEdition === 'COMMUNITY' || deploymentEdition === 'ENTERPRISE'- const { id, name, data_source_type, display_status } = detail+ const { id, name, data_source_type, data_source_info, display_status } = detailconst enabled = detail.enabled ?? falseconst archived = detail.archived ?? falseconst { t } = useTranslation(['common', 'datasetDocuments'])@@ -91,7 +93,8 @@ const Operations = ({const canShowRenameAction = canEdit && !archivedconst canShowSummaryAction = canEdit && !archived && isNonCloudEditionconst canShowSettingsAction = canViewSettings- const canShowDownloadAction = canDownload && data_source_type === DataSourceType.FILE+ const canShowDownloadAction =+ canDownload && isDownloadableFileDocument(data_source_type, data_source_info)const canShowSyncAction =canEdit &&!archived &&diff --git a/web/app/components/datasets/documents/detail/index.tsx b/web/app/components/datasets/documents/detail/index.tsxindex cadc5a43913..ec2a7c3de0c 100644--- a/web/app/components/datasets/documents/detail/index.tsx+++ b/web/app/components/datasets/documents/detail/index.tsx@@ -282,6 +282,7 @@ const DocumentDetail: FC<DocumentDetailProps> = ({ datasetId, documentId }) => {archived: documentDetail?.archived || false,id: documentId,data_source_type: documentDetail?.data_source_type || '',+ data_source_info: documentDetail?.data_source_info,doc_form: documentDetail?.doc_form || '',}),[@@ -290,6 +291,7 @@ const DocumentDetail: FC<DocumentDetailProps> = ({ datasetId, documentId }) => {documentDetail?.archived,documentId,documentDetail?.data_source_type,+ documentDetail?.data_source_info,documentDetail?.doc_form,],)diff --git a/web/app/components/datasets/documents/document-source.ts b/web/app/components/datasets/documents/document-source.tsindex 575b00b4330..6bc07d5cbad 100644--- a/web/app/components/datasets/documents/document-source.ts+++ b/web/app/components/datasets/documents/document-source.ts@@ -1,4 +1,6 @@import type { DataSourceNotionPage } from '@/models/common'+import { DataSourceType } from '@/models/datasets'+import { DatasourceType } from '@/models/pipeline'// data_source_info is written by the data source that imported the document, so its keys are// read defensively for each source type.@@ -12,6 +14,21 @@ const getString = (record: Record<string, unknown> | undefined, key: string) =>return typeof value === 'string' ? value : undefined}+/** Whether a document source has a file reference supported by the download API. */+export const isDownloadableFileDocument = (+ sourceType: string | null | undefined,+ info: DocumentSourceInfo,+) => {+ if (sourceType === DataSourceType.FILE) return true+ if (sourceType !== DatasourceType.localFile) return false++ return Boolean(+ getString(info ?? undefined, 'real_file_id') ||+ getString(info ?? undefined, 'related_id') ||+ getString(info ?? undefined, 'reference'),+ )+}+const isNotionPageIcon = (value: unknown,): value is NonNullable<DataSourceNotionPage['page_icon']> =>