Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

15.1 KB
Raw Blame
'use client'

import type { FC, MouseEvent } from 'react'
import type { StepTwoProps } from './types'
import { RadioGroup } from '@langgenius/dify-ui/radio-group'
import { Separator } from '@langgenius/dify-ui/separator'
import { useMediaQuery } from '@langgenius/dify-ui/use-media-query'
import { useMutation } from '@tanstack/react-query'
import { useCallback, useEffect, useRef, useState } from 'react'
import { useTranslation } from 'react-i18next'
import { useLocale } from '#i18n'
import {
  toChunkingMode,
  toCreationDataSourceType,
  toRetrievalConfig,
  toSummaryIndexSetting,
} from '@/app/components/datasets/dataset-fields'
import { toast } from '@/app/notifications'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { LanguagesSupported } from '@/i18n/language'
import { DataSourceProvider } from '@/models/common'
import { ChunkingMode, ProcessMode } from '@/models/datasets'
import { consoleQuery } from '@/service/console'
import {
  GeneralChunkingOptions,
  IndexingModeSection,
  ParentChildOptions,
  PreviewPanel,
  StepTwoFooter,
} from './components'
import {
  IndexingType,
  MAXIMUM_CHUNK_TOKEN_LENGTH,
  useDocumentCreation,
  useIndexingConfig,
  useIndexingEstimate,
  usePreviewState,
  useSegmentationState,
} from './hooks'
import { isProcessMode, toSegmentationRules } from './process-rule'

// oxlint-disable-next-line no-barrel-files/no-barrel-files
export { IndexingType }

const StepTwo: FC<StepTwoProps> = ({
  isSetting,
  documentDetail,
  isAPIKeySet,
  datasetId,
  indexingType: propsIndexingType,
  dataSourceType: inCreatePageDataSourceType,
  files,
  notionPages = [],
  notionCredentialId,
  websitePages = [],
  crawlOptions,
  websiteCrawlProvider = DataSourceProvider.jinaReader,
  websiteCrawlJobId = '',
  canCreateDocument = true,
  onStepChange,
  updateIndexingTypeCache,
  updateResultCache,
  onSave,
  onCancel,
  updateRetrievalMethodCache,
}) => {
  const generalChunkingModes: readonly ChunkingMode[] = [ChunkingMode.text, ChunkingMode.qa]

  const { t } = useTranslation(['datasetCreation'])
  const locale = useLocale()
  const isBelowSm = useMediaQuery('(width < 40rem)', {})
  const [isPreviewOpen, setIsPreviewOpen] = useState(false)
  const previewTriggerRef = useRef<HTMLButtonElement | null>(null)
  const currentDataset = useDatasetDetailContextWithSelector((s) => s.dataset)
  const mutateDatasetRes = useDatasetDetailContextWithSelector((s) => s.mutateDatasetRes)

  // Computed flags
  const isInUpload = Boolean(currentDataset)
  const isUploadInEmptyDataset = isInUpload && !currentDataset?.doc_form
  const isNotUploadInEmptyDataset = !isUploadInEmptyDataset
  const isInInit = !isInUpload && !isSetting
  const isInCreatePage = !datasetId || (datasetId && !currentDataset?.data_source_type)
  const dataSourceType = isInCreatePage
    ? inCreatePageDataSourceType
    : (toCreationDataSourceType(currentDataset?.data_source_type) ?? inCreatePageDataSourceType)
  const hasSetIndexType = !!propsIndexingType
  const isModelAndRetrievalConfigDisabled = !!datasetId && !!currentDataset?.data_source_type

  // Document form state
  const [docForm, setDocForm] = useState<ChunkingMode>(
    (datasetId && toChunkingMode(documentDetail?.doc_form)) || ChunkingMode.text,
  )
  const [docLanguage, setDocLanguage] = useState<string>(() =>
    datasetId && documentDetail?.doc_language
      ? documentDetail.doc_language
      : locale !== LanguagesSupported[1]
        ? 'English'
        : 'Chinese Simplified',
  )
  const [isQAConfirmDialogOpen, setIsQAConfirmDialogOpen] = useState(false)
  const datasetChunkingMode = toChunkingMode(currentDataset?.doc_form)
  const currentDocForm = datasetChunkingMode || docForm

  // Custom hooks
  const segmentation = useSegmentationState({
    initialSegmentationType:
      datasetChunkingMode === ChunkingMode.parentChild
        ? ProcessMode.parentChild
        : ProcessMode.general,
    initialSummaryIndexSetting:
      currentDataset?.summary_index_setting &&
      toSummaryIndexSetting(currentDataset.summary_index_setting),
  })
  const showSummaryIndexSetting = !currentDataset
  const indexing = useIndexingConfig({
    initialIndexType: propsIndexingType,
    initialEmbeddingModel: currentDataset?.embedding_model
      ? {
          provider: currentDataset.embedding_model_provider ?? '',
          model: currentDataset.embedding_model,
        }
      : undefined,
    initialRetrievalConfig:
      currentDataset && toRetrievalConfig(currentDataset.retrieval_model_dict),
    isAPIKeySet,
    hasSetIndexType,
  })
  const preview = usePreviewState({
    dataSourceType,
    files,
    notionPages,
    websitePages,
    documentDetail,
    datasetId,
  })
  const creation = useDocumentCreation({
    datasetId,
    isSetting,
    documentDetail,
    dataSourceType,
    files,
    notionPages,
    notionCredentialId,
    websitePages,
    crawlOptions,
    websiteCrawlProvider,
    websiteCrawlJobId,
    canCreateDocument,
    onStepChange,
    updateIndexingTypeCache,
    updateResultCache,
    updateRetrievalMethodCache,
    onSave,
    mutateDatasetRes,
  })
  const estimateHook = useIndexingEstimate({
    dataSourceType,
    datasetId,
    currentDocForm,
    docLanguage,
    files,
    previewItemId: preview.getPreviewPickerValue()?.id,
    notionPages,
    notionCredentialId,
    websitePages,
    crawlOptions,
    websiteCrawlProvider,
    websiteCrawlJobId,
    indexingTechnique: indexing.getIndexingTechnique() as IndexingType,
    processRule: segmentation.getProcessRule(currentDocForm),
  })

  // Fetch default process rule
  const fetchDefaultProcessRuleMutation = useMutation({
    ...consoleQuery.datasets.processRule.get.mutationOptions(),
    onSuccess(data) {
      const defaultRules = data.rules ? toSegmentationRules(data.rules) : undefined
      if (defaultRules) {
        segmentation.setSegmentIdentifier(defaultRules.segmentation.separator)
        segmentation.setMaxChunkLength(defaultRules.segmentation.max_tokens)
        segmentation.setOverlap(defaultRules.segmentation.chunk_overlap!)
        segmentation.setRules(defaultRules.pre_processing_rules)
        segmentation.setDefaultConfig(defaultRules)
      }
      segmentation.setLimitMaxChunkLength(data.limits.indexing_max_segmentation_tokens_length)
    },
  })

  // Event handlers
  const handleDocFormChange = useCallback(
    (value: ChunkingMode) => {
      if (value === ChunkingMode.qa && indexing.indexType === IndexingType.ECONOMICAL) {
        setIsQAConfirmDialogOpen(true)
        return
      }
      if (value === ChunkingMode.parentChild && indexing.indexType === IndexingType.ECONOMICAL)
        indexing.setIndexType(IndexingType.QUALIFIED)
      setDocForm(value)
      segmentation.setSegmentationType(
        value === ChunkingMode.parentChild ? ProcessMode.parentChild : ProcessMode.general,
      )
      estimateHook.reset()
    },
    [indexing, segmentation, estimateHook],
  )

  const updatePreview = useCallback(
    (event: MouseEvent<HTMLButtonElement>) => {
      if (
        segmentation.segmentationType === ProcessMode.general &&
        segmentation.maxChunkLength > MAXIMUM_CHUNK_TOKEN_LENGTH
      ) {
        toast.error(
          t(($) => $['stepTwo.maxLengthCheck'], {
            ns: 'datasetCreation',
            limit: MAXIMUM_CHUNK_TOKEN_LENGTH,
          }),
        )
        return
      }
      previewTriggerRef.current = event.currentTarget
      setIsPreviewOpen(true)
      estimateHook.fetchEstimate()
    },
    [segmentation, t, estimateHook],
  )

  const handleCreate = useCallback(async () => {
    if (!canCreateDocument) return

    const isValid = creation.validateParams({
      segmentationType: segmentation.segmentationType,
      maxChunkLength: segmentation.maxChunkLength,
      limitMaxChunkLength: segmentation.limitMaxChunkLength,
      overlap: segmentation.overlap,
      indexType: indexing.indexType,
      embeddingModel: indexing.embeddingModel,
      rerankModelList: indexing.rerankModelList,
      retrievalConfig: indexing.retrievalConfig,
    })
    if (!isValid) return
    const params = creation.buildCreationParams(
      currentDocForm,
      docLanguage,
      segmentation.getProcessRule(currentDocForm),
      indexing.retrievalConfig,
      indexing.embeddingModel,
      indexing.getIndexingTechnique(),
      segmentation.summaryIndexSetting,
    )
    if (!params) return
    try {
      await creation.executeCreation(params, indexing.indexType, indexing.retrievalConfig)
    } catch {
      // The service reports request failures; keep this step and its draft available for retry.
    }
  }, [canCreateDocument, creation, segmentation, indexing, currentDocForm, docLanguage])

  const handlePickerChange = useCallback(
    (selected: { id: string; name: string }) => {
      estimateHook.reset()
      preview.handlePreviewChange(selected)
      estimateHook.fetchEstimate(selected.id)
    },
    [estimateHook, preview],
  )

  const handleQAConfirm = useCallback(() => {
    setIsQAConfirmDialogOpen(false)
    indexing.setIndexType(IndexingType.QUALIFIED)
    setDocForm(ChunkingMode.qa)
  }, [indexing])

  // Initialize rules
  useEffect(() => {
    if (!isSetting) {
      fetchDefaultProcessRuleMutation.mutate({})
    } else if (documentDetail?.dataset_process_rule) {
      const { mode, rules } = documentDetail.dataset_process_rule
      const segmentationRules = rules ? toSegmentationRules(rules) : undefined
      if (segmentationRules) {
        const isHierarchical =
          documentDetail.doc_form === ChunkingMode.parentChild ||
          Boolean(segmentationRules.parent_mode && segmentationRules.subchunk_segmentation)
        segmentation.applyConfigFromRules(segmentationRules, isHierarchical)
      }
      if (isProcessMode(mode)) segmentation.setSegmentationType(mode)
    }
    // oxlint-disable-next-line react/exhaustive-deps
  }, [])

  // Show options conditions
  const showGeneralOption =
    (isInUpload && !!datasetChunkingMode && generalChunkingModes.includes(datasetChunkingMode)) ||
    isUploadInEmptyDataset ||
    isInInit
  const showParentChildOption =
    (isInUpload && datasetChunkingMode === ChunkingMode.parentChild) ||
    isUploadInEmptyDataset ||
    isInInit

  return (
    <div className="flex size-full min-w-0 max-sm:flex-col max-sm:overflow-y-auto">
      <div className="@container/chunkfields relative min-w-0 py-6 max-sm:w-full max-sm:shrink-0 max-sm:px-4 sm:h-full sm:w-1/2 sm:overflow-y-auto sm:px-12">
        <h1 className="mb-1 system-md-semibold text-text-secondary">
          {t(($) => $['stepTwo.segmentation'], { ns: 'datasetCreation' })}
        </h1>
        <RadioGroup<ChunkingMode>
          className="block"
          aria-label={t(($) => $['stepTwo.segmentation'], { ns: 'datasetCreation' })}
          value={currentDocForm === ChunkingMode.qa ? ChunkingMode.text : currentDocForm}
          onValueChange={handleDocFormChange}
        >
          {showGeneralOption && (
            <GeneralChunkingOptions
              segmentIdentifier={segmentation.segmentIdentifier}
              maxChunkLength={segmentation.maxChunkLength}
              overlap={segmentation.overlap}
              rules={segmentation.rules}
              currentDocForm={currentDocForm}
              docLanguage={docLanguage}
              isActive={generalChunkingModes.includes(currentDocForm)}
              isInUpload={isInUpload}
              isNotUploadInEmptyDataset={isNotUploadInEmptyDataset}
              hasCurrentDatasetDocForm={!!currentDataset?.doc_form}
              onSegmentIdentifierChange={(value) => segmentation.setSegmentIdentifier(value, true)}
              onMaxChunkLengthChange={segmentation.setMaxChunkLength}
              onOverlapChange={segmentation.setOverlap}
              onRuleToggle={segmentation.toggleRule}
              onDocFormChange={handleDocFormChange}
              onDocLanguageChange={setDocLanguage}
              onPreview={updatePreview}
              onReset={segmentation.resetToDefaults}
              locale={locale}
              showSummaryIndexSetting={showSummaryIndexSetting}
              summaryIndexSetting={segmentation.summaryIndexSetting}
              onSummaryIndexSettingChange={segmentation.handleSummaryIndexSettingChange}
            />
          )}
          {showParentChildOption && (
            <ParentChildOptions
              parentChildConfig={segmentation.parentChildConfig}
              rules={segmentation.rules}
              currentDocForm={currentDocForm}
              isActive={currentDocForm === ChunkingMode.parentChild}
              isInUpload={isInUpload}
              isNotUploadInEmptyDataset={isNotUploadInEmptyDataset}
              onChunkForContextChange={segmentation.setChunkForContext}
              onParentDelimiterChange={(v) => segmentation.updateParentConfig('delimiter', v)}
              onParentMaxLengthChange={(v) => segmentation.updateParentConfig('maxLength', v)}
              onChildDelimiterChange={(v) => segmentation.updateChildConfig('delimiter', v)}
              onChildMaxLengthChange={(v) => segmentation.updateChildConfig('maxLength', v)}
              onRuleToggle={segmentation.toggleRule}
              onPreview={updatePreview}
              onReset={segmentation.resetToDefaults}
              showSummaryIndexSetting={showSummaryIndexSetting}
              summaryIndexSetting={segmentation.summaryIndexSetting}
              onSummaryIndexSettingChange={segmentation.handleSummaryIndexSettingChange}
            />
          )}
        </RadioGroup>
        <Separator className="my-5 h-[0.5px]" />
        <IndexingModeSection
          indexType={indexing.indexType}
          hasSetIndexType={hasSetIndexType}
          docForm={docForm}
          embeddingModel={indexing.embeddingModel}
          embeddingModelList={indexing.embeddingModelList}
          retrievalConfig={indexing.retrievalConfig}
          showMultiModalTip={indexing.showMultiModalTip}
          isModelAndRetrievalConfigDisabled={isModelAndRetrievalConfigDisabled}
          datasetId={datasetId}
          isQAConfirmDialogOpen={isQAConfirmDialogOpen}
          onIndexTypeChange={indexing.setIndexType}
          onEmbeddingModelChange={indexing.setEmbeddingModel}
          onRetrievalConfigChange={indexing.setRetrievalConfig}
          onQAConfirmDialogClose={() => setIsQAConfirmDialogOpen(false)}
          onQAConfirmDialogConfirm={handleQAConfirm}
        />
        <StepTwoFooter
          isSetting={isSetting}
          isCreating={creation.isCreating}
          onPrevious={() => onStepChange?.(-1)}
          onCreate={handleCreate}
          onCancel={onCancel}
        />
      </div>
      <PreviewPanel
        isBelowSm={isBelowSm}
        isOpen={isPreviewOpen}
        onClose={() => setIsPreviewOpen(false)}
        finalFocus={previewTriggerRef}
        dataSourceType={dataSourceType}
        currentDocForm={currentDocForm}
        estimate={estimateHook.estimate}
        parentChildConfig={segmentation.parentChildConfig}
        isSetting={isSetting}
        pickerFiles={preview.getPreviewPickerItems()}
        pickerValue={preview.getPreviewPickerValue()}
        isIdle={estimateHook.isIdle}
        isPending={estimateHook.isPending}
        onPickerChange={handlePickerChange}
      />
    </div>
  )
}

export default StepTwo