Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
'use client'
import type { FC, MouseEvent } from 'react'
import type { StepTwoProps } from './types'
import { RadioGroup } from '@langgenius/dify-ui/radio-group'
import { Separator } from '@langgenius/dify-ui/separator'
import { useMediaQuery } from '@langgenius/dify-ui/use-media-query'
import { useMutation } from '@tanstack/react-query'
import { useCallback, useEffect, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useLocale } from '#i18n'
import {
toChunkingMode,
toCreationDataSourceType,
toRetrievalConfig,
toSummaryIndexSetting,
} from '@/app/components/datasets/dataset-fields'
import { toast } from '@/app/notifications'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { LanguagesSupported } from '@/i18n/language'
import { DataSourceProvider } from '@/models/common'
import { ChunkingMode, ProcessMode } from '@/models/datasets'
import { consoleQuery } from '@/service/console'
import {
GeneralChunkingOptions,
IndexingModeSection,
ParentChildOptions,
PreviewPanel,
StepTwoFooter,
} from './components'
import {
IndexingType,
MAXIMUM_CHUNK_TOKEN_LENGTH,
useDocumentCreation,
useIndexingConfig,
useIndexingEstimate,
usePreviewState,
useSegmentationState,
} from './hooks'
import { isProcessMode, toSegmentationRules } from './process-rule'
// oxlint-disable-next-line no-barrel-files/no-barrel-files
export { IndexingType }
const StepTwo: FC<StepTwoProps> = ({
isSetting,
documentDetail,
isAPIKeySet,
datasetId,
indexingType: propsIndexingType,
dataSourceType: inCreatePageDataSourceType,
files,
notionPages = [],
notionCredentialId,
websitePages = [],
crawlOptions,
websiteCrawlProvider = DataSourceProvider.jinaReader,
websiteCrawlJobId = '',
canCreateDocument = true,
onStepChange,
updateIndexingTypeCache,
updateResultCache,
onSave,
onCancel,
updateRetrievalMethodCache,
}) => {
const generalChunkingModes: readonly ChunkingMode[] = [ChunkingMode.text, ChunkingMode.qa]
const { t } = useTranslation(['datasetCreation'])
const locale = useLocale()
const isBelowSm = useMediaQuery('(width < 40rem)', {})
const [isPreviewOpen, setIsPreviewOpen] = useState(false)
const previewTriggerRef = useRef<HTMLButtonElement | null>(null)
const currentDataset = useDatasetDetailContextWithSelector((s) => s.dataset)
const mutateDatasetRes = useDatasetDetailContextWithSelector((s) => s.mutateDatasetRes)
// Computed flags
const isInUpload = Boolean(currentDataset)
const isUploadInEmptyDataset = isInUpload && !currentDataset?.doc_form
const isNotUploadInEmptyDataset = !isUploadInEmptyDataset
const isInInit = !isInUpload && !isSetting
const isInCreatePage = !datasetId || (datasetId && !currentDataset?.data_source_type)
const dataSourceType = isInCreatePage
? inCreatePageDataSourceType
: (toCreationDataSourceType(currentDataset?.data_source_type) ?? inCreatePageDataSourceType)
const hasSetIndexType = !!propsIndexingType
const isModelAndRetrievalConfigDisabled = !!datasetId && !!currentDataset?.data_source_type
// Document form state
const [docForm, setDocForm] = useState<ChunkingMode>(
(datasetId && toChunkingMode(documentDetail?.doc_form)) || ChunkingMode.text,
)
const [docLanguage, setDocLanguage] = useState<string>(() =>
datasetId && documentDetail?.doc_language
? documentDetail.doc_language
: locale !== LanguagesSupported[1]
? 'English'
: 'Chinese Simplified',
)
const [isQAConfirmDialogOpen, setIsQAConfirmDialogOpen] = useState(false)
const datasetChunkingMode = toChunkingMode(currentDataset?.doc_form)
const currentDocForm = datasetChunkingMode || docForm
// Custom hooks
const segmentation = useSegmentationState({
initialSegmentationType:
datasetChunkingMode === ChunkingMode.parentChild
? ProcessMode.parentChild
: ProcessMode.general,
initialSummaryIndexSetting:
currentDataset?.summary_index_setting &&
toSummaryIndexSetting(currentDataset.summary_index_setting),
})
const showSummaryIndexSetting = !currentDataset
const indexing = useIndexingConfig({
initialIndexType: propsIndexingType,
initialEmbeddingModel: currentDataset?.embedding_model
? {
provider: currentDataset.embedding_model_provider ?? '',
model: currentDataset.embedding_model,
}
: undefined,
initialRetrievalConfig:
currentDataset && toRetrievalConfig(currentDataset.retrieval_model_dict),
isAPIKeySet,
hasSetIndexType,
})
const preview = usePreviewState({
dataSourceType,
files,
notionPages,
websitePages,
documentDetail,
datasetId,
})
const creation = useDocumentCreation({
datasetId,
isSetting,
documentDetail,
dataSourceType,
files,
notionPages,
notionCredentialId,
websitePages,
crawlOptions,
websiteCrawlProvider,
websiteCrawlJobId,
canCreateDocument,
onStepChange,
updateIndexingTypeCache,
updateResultCache,
updateRetrievalMethodCache,
onSave,
mutateDatasetRes,
})
const estimateHook = useIndexingEstimate({
dataSourceType,
datasetId,
currentDocForm,
docLanguage,
files,
previewItemId: preview.getPreviewPickerValue()?.id,
notionPages,
notionCredentialId,
websitePages,
crawlOptions,
websiteCrawlProvider,
websiteCrawlJobId,
indexingTechnique: indexing.getIndexingTechnique() as IndexingType,
processRule: segmentation.getProcessRule(currentDocForm),
})
// Fetch default process rule
const fetchDefaultProcessRuleMutation = useMutation({
...consoleQuery.datasets.processRule.get.mutationOptions(),
onSuccess(data) {
const defaultRules = data.rules ? toSegmentationRules(data.rules) : undefined
if (defaultRules) {
segmentation.setSegmentIdentifier(defaultRules.segmentation.separator)
segmentation.setMaxChunkLength(defaultRules.segmentation.max_tokens)
segmentation.setOverlap(defaultRules.segmentation.chunk_overlap!)
segmentation.setRules(defaultRules.pre_processing_rules)
segmentation.setDefaultConfig(defaultRules)
}
segmentation.setLimitMaxChunkLength(data.limits.indexing_max_segmentation_tokens_length)
},
})
// Event handlers
const handleDocFormChange = useCallback(
(value: ChunkingMode) => {
if (value === ChunkingMode.qa && indexing.indexType === IndexingType.ECONOMICAL) {
setIsQAConfirmDialogOpen(true)
return
}
if (value === ChunkingMode.parentChild && indexing.indexType === IndexingType.ECONOMICAL)
indexing.setIndexType(IndexingType.QUALIFIED)
setDocForm(value)
segmentation.setSegmentationType(
value === ChunkingMode.parentChild ? ProcessMode.parentChild : ProcessMode.general,
)
estimateHook.reset()
},
[indexing, segmentation, estimateHook],
)
const updatePreview = useCallback(
(event: MouseEvent<HTMLButtonElement>) => {
if (
segmentation.segmentationType === ProcessMode.general &&
segmentation.maxChunkLength > MAXIMUM_CHUNK_TOKEN_LENGTH
) {
toast.error(
t(($) => $['stepTwo.maxLengthCheck'], {
ns: 'datasetCreation',
limit: MAXIMUM_CHUNK_TOKEN_LENGTH,
}),
)
return
}
previewTriggerRef.current = event.currentTarget
setIsPreviewOpen(true)
estimateHook.fetchEstimate()
},
[segmentation, t, estimateHook],
)
const handleCreate = useCallback(async () => {
if (!canCreateDocument) return
const isValid = creation.validateParams({
segmentationType: segmentation.segmentationType,
maxChunkLength: segmentation.maxChunkLength,
limitMaxChunkLength: segmentation.limitMaxChunkLength,
overlap: segmentation.overlap,
indexType: indexing.indexType,
embeddingModel: indexing.embeddingModel,
rerankModelList: indexing.rerankModelList,
retrievalConfig: indexing.retrievalConfig,
})
if (!isValid) return
const params = creation.buildCreationParams(
currentDocForm,
docLanguage,
segmentation.getProcessRule(currentDocForm),
indexing.retrievalConfig,
indexing.embeddingModel,
indexing.getIndexingTechnique(),
segmentation.summaryIndexSetting,
)
if (!params) return
try {
await creation.executeCreation(params, indexing.indexType, indexing.retrievalConfig)
} catch {
// The service reports request failures; keep this step and its draft available for retry.
}
}, [canCreateDocument, creation, segmentation, indexing, currentDocForm, docLanguage])
const handlePickerChange = useCallback(
(selected: { id: string; name: string }) => {
estimateHook.reset()
preview.handlePreviewChange(selected)
estimateHook.fetchEstimate(selected.id)
},
[estimateHook, preview],
)
const handleQAConfirm = useCallback(() => {
setIsQAConfirmDialogOpen(false)
indexing.setIndexType(IndexingType.QUALIFIED)
setDocForm(ChunkingMode.qa)
}, [indexing])
// Initialize rules
useEffect(() => {
if (!isSetting) {
fetchDefaultProcessRuleMutation.mutate({})
} else if (documentDetail?.dataset_process_rule) {
const { mode, rules } = documentDetail.dataset_process_rule
const segmentationRules = rules ? toSegmentationRules(rules) : undefined
if (segmentationRules) {
const isHierarchical =
documentDetail.doc_form === ChunkingMode.parentChild ||
Boolean(segmentationRules.parent_mode && segmentationRules.subchunk_segmentation)
segmentation.applyConfigFromRules(segmentationRules, isHierarchical)
}
if (isProcessMode(mode)) segmentation.setSegmentationType(mode)
}
// oxlint-disable-next-line react/exhaustive-deps
}, [])
// Show options conditions
const showGeneralOption =
(isInUpload && !!datasetChunkingMode && generalChunkingModes.includes(datasetChunkingMode)) ||
isUploadInEmptyDataset ||
isInInit
const showParentChildOption =
(isInUpload && datasetChunkingMode === ChunkingMode.parentChild) ||
isUploadInEmptyDataset ||
isInInit
return (
<div className="flex size-full min-w-0 max-sm:flex-col max-sm:overflow-y-auto">
<div className="@container/chunkfields relative min-w-0 py-6 max-sm:w-full max-sm:shrink-0 max-sm:px-4 sm:h-full sm:w-1/2 sm:overflow-y-auto sm:px-12">
<h1 className="mb-1 system-md-semibold text-text-secondary">
{t(($) => $['stepTwo.segmentation'], { ns: 'datasetCreation' })}
</h1>
<RadioGroup<ChunkingMode>
className="block"
aria-label={t(($) => $['stepTwo.segmentation'], { ns: 'datasetCreation' })}
value={currentDocForm === ChunkingMode.qa ? ChunkingMode.text : currentDocForm}
onValueChange={handleDocFormChange}
>
{showGeneralOption && (
<GeneralChunkingOptions
segmentIdentifier={segmentation.segmentIdentifier}
maxChunkLength={segmentation.maxChunkLength}
overlap={segmentation.overlap}
rules={segmentation.rules}
currentDocForm={currentDocForm}
docLanguage={docLanguage}
isActive={generalChunkingModes.includes(currentDocForm)}
isInUpload={isInUpload}
isNotUploadInEmptyDataset={isNotUploadInEmptyDataset}
hasCurrentDatasetDocForm={!!currentDataset?.doc_form}
onSegmentIdentifierChange={(value) => segmentation.setSegmentIdentifier(value, true)}
onMaxChunkLengthChange={segmentation.setMaxChunkLength}
onOverlapChange={segmentation.setOverlap}
onRuleToggle={segmentation.toggleRule}
onDocFormChange={handleDocFormChange}
onDocLanguageChange={setDocLanguage}
onPreview={updatePreview}
onReset={segmentation.resetToDefaults}
locale={locale}
showSummaryIndexSetting={showSummaryIndexSetting}
summaryIndexSetting={segmentation.summaryIndexSetting}
onSummaryIndexSettingChange={segmentation.handleSummaryIndexSettingChange}
/>
)}
{showParentChildOption && (
<ParentChildOptions
parentChildConfig={segmentation.parentChildConfig}
rules={segmentation.rules}
currentDocForm={currentDocForm}
isActive={currentDocForm === ChunkingMode.parentChild}
isInUpload={isInUpload}
isNotUploadInEmptyDataset={isNotUploadInEmptyDataset}
onChunkForContextChange={segmentation.setChunkForContext}
onParentDelimiterChange={(v) => segmentation.updateParentConfig('delimiter', v)}
onParentMaxLengthChange={(v) => segmentation.updateParentConfig('maxLength', v)}
onChildDelimiterChange={(v) => segmentation.updateChildConfig('delimiter', v)}
onChildMaxLengthChange={(v) => segmentation.updateChildConfig('maxLength', v)}
onRuleToggle={segmentation.toggleRule}
onPreview={updatePreview}
onReset={segmentation.resetToDefaults}
showSummaryIndexSetting={showSummaryIndexSetting}
summaryIndexSetting={segmentation.summaryIndexSetting}
onSummaryIndexSettingChange={segmentation.handleSummaryIndexSettingChange}
/>
)}
</RadioGroup>
<Separator className="my-5 h-[0.5px]" />
<IndexingModeSection
indexType={indexing.indexType}
hasSetIndexType={hasSetIndexType}
docForm={docForm}
embeddingModel={indexing.embeddingModel}
embeddingModelList={indexing.embeddingModelList}
retrievalConfig={indexing.retrievalConfig}
showMultiModalTip={indexing.showMultiModalTip}
isModelAndRetrievalConfigDisabled={isModelAndRetrievalConfigDisabled}
datasetId={datasetId}
isQAConfirmDialogOpen={isQAConfirmDialogOpen}
onIndexTypeChange={indexing.setIndexType}
onEmbeddingModelChange={indexing.setEmbeddingModel}
onRetrievalConfigChange={indexing.setRetrievalConfig}
onQAConfirmDialogClose={() => setIsQAConfirmDialogOpen(false)}
onQAConfirmDialogConfirm={handleQAConfirm}
/>
<StepTwoFooter
isSetting={isSetting}
isCreating={creation.isCreating}
onPrevious={() => onStepChange?.(-1)}
onCreate={handleCreate}
onCancel={onCancel}
/>
</div>
<PreviewPanel
isBelowSm={isBelowSm}
isOpen={isPreviewOpen}
onClose={() => setIsPreviewOpen(false)}
finalFocus={previewTriggerRef}
dataSourceType={dataSourceType}
currentDocForm={currentDocForm}
estimate={estimateHook.estimate}
parentChildConfig={segmentation.parentChildConfig}
isSetting={isSetting}
pickerFiles={preview.getPreviewPickerItems()}
pickerValue={preview.getPreviewPickerValue()}
isIdle={estimateHook.isIdle}
isPending={estimateHook.isPending}
onPickerChange={handlePickerChange}
/>
</div>
)
}
export default StepTwo