Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

9.2 KB
Raw Blame
import type {
  DatasetAndDocumentResponse,
  DocumentDetailResponse,
} from '@dify/contracts/api/console/datasets/types.gen'
import type { ProviderWithModelsResponse } from '@dify/contracts/api/console/workspaces/types.gen'
import type { StepTwoFile, StepTwoNotionPage } from '../types'
import type { DefaultModel } from '@/app/components/header/account-setting/model-provider-page/declarations'
import type {
  ChunkingMode,
  CrawlOptions,
  CrawlResultItem,
  CreateDocumentReq,
  ProcessRule,
  SummaryIndexSetting as SummaryIndexSettingType,
} from '@/models/datasets'
import type { RetrievalConfig, RETRIEVE_METHOD } from '@/types/app'
import { useMutation, useQueryClient } from '@tanstack/react-query'
import { useCallback } from 'react'
import { useTranslation } from 'react-i18next'
import { trackEvent } from '@/app/components/base/amplitude'
import { isReRankModelSelected } from '@/app/components/datasets/common/check-rerank-model'
import { toast } from '@/app/notifications'
import { DataSourceProvider } from '@/models/common'
import { DataSourceType, ProcessMode } from '@/models/datasets'
import { consoleQuery } from '@/service/console'
import { getNotionInfo, getWebsiteInfo } from '../creation-params'
import { IndexingType } from './use-indexing-config'
import { MAXIMUM_CHUNK_TOKEN_LENGTH } from './use-segmentation-state'

type UseDocumentCreationOptions = {
  datasetId?: string
  isSetting?: boolean
  documentDetail?: DocumentDetailResponse
  dataSourceType: DataSourceType
  files: StepTwoFile[]
  notionPages: StepTwoNotionPage[]
  notionCredentialId: string
  websitePages: CrawlResultItem[]
  crawlOptions?: CrawlOptions
  websiteCrawlProvider?: DataSourceProvider
  websiteCrawlJobId?: string
  canCreateDocument?: boolean
  // Callbacks
  onStepChange?: (delta: number) => void
  updateIndexingTypeCache?: (type: string) => void
  updateResultCache?: (res: DatasetAndDocumentResponse) => void
  updateRetrievalMethodCache?: (method: RETRIEVE_METHOD | '') => void
  onSave?: () => void
  mutateDatasetRes?: () => void
}
type ValidationParams = {
  segmentationType: ProcessMode
  maxChunkLength: number
  limitMaxChunkLength: number
  overlap: number
  indexType: IndexingType
  embeddingModel: DefaultModel
  rerankModelList: ProviderWithModelsResponse[]
  retrievalConfig: RetrievalConfig
}
export const useDocumentCreation = (options: UseDocumentCreationOptions) => {
  const { t } = useTranslation(['appDebug', 'datasetCreation'])
  const {
    datasetId,
    isSetting,
    documentDetail,
    dataSourceType,
    files,
    notionPages,
    notionCredentialId,
    websitePages,
    crawlOptions,
    websiteCrawlProvider = DataSourceProvider.jinaReader,
    websiteCrawlJobId = '',
    canCreateDocument = true,
    onStepChange,
    updateIndexingTypeCache,
    updateResultCache,
    updateRetrievalMethodCache,
    onSave,
    mutateDatasetRes,
  } = options
  const { mutateAsync: createFirstDocument, isPending: isCreatingFirstDocument } = useMutation(
    consoleQuery.datasets.init.post.mutationOptions(),
  )
  const { mutateAsync: createDocument, isPending: isCreatingDocument } = useMutation(
    consoleQuery.datasets.byDatasetId.documents.post.mutationOptions(),
  )
  const queryClient = useQueryClient()
  const invalidDatasetList = useCallback(() => {
    queryClient.invalidateQueries({ queryKey: consoleQuery.datasets.get.key() })
  }, [queryClient])
  const isCreating = isCreatingFirstDocument || isCreatingDocument
  // Validate creation params
  const validateParams = useCallback(
    (params: ValidationParams): boolean => {
      const {
        segmentationType,
        maxChunkLength,
        limitMaxChunkLength,
        overlap,
        indexType,
        embeddingModel,
        rerankModelList,
        retrievalConfig,
      } = params
      if (segmentationType === ProcessMode.general && overlap > maxChunkLength) {
        toast.error(t(($) => $['stepTwo.overlapCheck'], { ns: 'datasetCreation' }))
        return false
      }
      if (segmentationType === ProcessMode.general && maxChunkLength > limitMaxChunkLength) {
        toast.error(
          t(($) => $['stepTwo.maxLengthCheck'], {
            ns: 'datasetCreation',
            limit: limitMaxChunkLength,
          }),
        )
        return false
      }
      if (!isSetting) {
        if (
          indexType === IndexingType.QUALIFIED &&
          (!embeddingModel.model || !embeddingModel.provider)
        ) {
          toast.error(t(($) => $['datasetConfig.embeddingModelRequired'], { ns: 'appDebug' }))
          return false
        }
        if (
          !isReRankModelSelected({
            rerankModelList,
            retrievalConfig,
            indexMethod: indexType,
          })
        ) {
          toast.error(t(($) => $['datasetConfig.rerankModelRequired'], { ns: 'appDebug' }))
          return false
        }
      }
      return true
    },
    [t, isSetting],
  )
  // Build creation params
  const buildCreationParams = useCallback(
    (
      currentDocForm: ChunkingMode,
      docLanguage: string,
      processRule: ProcessRule,
      retrievalConfig: RetrievalConfig,
      embeddingModel: DefaultModel,
      indexingTechnique: string,
      summaryIndexSetting?: SummaryIndexSettingType,
    ): CreateDocumentReq | null => {
      if (isSetting) {
        return {
          original_document_id: documentDetail?.id,
          doc_form: currentDocForm,
          doc_language: docLanguage,
          process_rule: processRule,
          summary_index_setting: summaryIndexSetting,
          retrieval_model: retrievalConfig,
          embedding_model: embeddingModel.model,
          embedding_model_provider: embeddingModel.provider,
          indexing_technique: indexingTechnique,
        } as CreateDocumentReq
      }
      const params: CreateDocumentReq = {
        data_source: {
          type: dataSourceType,
          info_list: {
            data_source_type: dataSourceType,
          },
        },
        indexing_technique: indexingTechnique,
        process_rule: processRule,
        summary_index_setting: summaryIndexSetting,
        doc_form: currentDocForm,
        doc_language: docLanguage,
        retrieval_model: retrievalConfig,
        embedding_model: embeddingModel.model,
        embedding_model_provider: embeddingModel.provider,
      } as CreateDocumentReq
      // Add data source specific info
      if (dataSourceType === DataSourceType.FILE) {
        params.data_source!.info_list.file_info_list = {
          file_ids: files.map((file) => file.id || '').filter(Boolean),
        }
      }
      if (dataSourceType === DataSourceType.NOTION)
        params.data_source!.info_list.notion_info_list = getNotionInfo(
          notionPages,
          notionCredentialId,
        )
      if (dataSourceType === DataSourceType.WEB) {
        params.data_source!.info_list.website_info_list = getWebsiteInfo({
          websiteCrawlProvider,
          websiteCrawlJobId,
          websitePages,
          crawlOptions,
        })
      }
      return params
    },
    [
      isSetting,
      documentDetail,
      dataSourceType,
      files,
      notionPages,
      notionCredentialId,
      websitePages,
      websiteCrawlProvider,
      websiteCrawlJobId,
      crawlOptions,
    ],
  )
  // Execute creation
  const executeCreation = useCallback(
    async (
      params: CreateDocumentReq,
      indexType: IndexingType,
      retrievalConfig: RetrievalConfig,
    ) => {
      if (datasetId && !isSetting && !canCreateDocument) return

      if (!datasetId) {
        await createFirstDocument(
          { body: params },
          {
            onSuccess(data) {
              updateIndexingTypeCache?.(indexType)
              updateResultCache?.(data)
              updateRetrievalMethodCache?.(retrievalConfig.search_method as RETRIEVE_METHOD)
            },
          },
        )
      } else {
        await createDocument(
          { params: { dataset_id: datasetId }, body: params },
          {
            onSuccess(data) {
              updateIndexingTypeCache?.(indexType)
              updateResultCache?.(data)
              updateRetrievalMethodCache?.(retrievalConfig.search_method as RETRIEVE_METHOD)
            },
          },
        )
      }
      mutateDatasetRes?.()
      invalidDatasetList()
      trackEvent('create_datasets', {
        data_source_type: dataSourceType,
        indexing_technique: indexType,
      })
      onStepChange?.(+1)
      if (isSetting) onSave?.()
    },
    [
      datasetId,
      createFirstDocument,
      createDocument,
      updateIndexingTypeCache,
      updateResultCache,
      updateRetrievalMethodCache,
      mutateDatasetRes,
      invalidDatasetList,
      dataSourceType,
      onStepChange,
      isSetting,
      canCreateDocument,
      onSave,
    ],
  )
  // Validate preview params
  const validatePreviewParams = useCallback(
    (maxChunkLength: number): boolean => {
      if (maxChunkLength > MAXIMUM_CHUNK_TOKEN_LENGTH) {
        toast.error(
          t(($) => $['stepTwo.maxLengthCheck'], {
            ns: 'datasetCreation',
            limit: MAXIMUM_CHUNK_TOKEN_LENGTH,
          }),
        )
        return false
      }
      return true
    },
    [t],
  )
  return {
    isCreating,
    validateParams,
    buildCreationParams,
    executeCreation,
    validatePreviewParams,
  }
}