Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

16.2 KB
Raw Blame
'use client'

import type { DatasourceProviderAuthResponse } from '@dify/contracts/api/console/auth/types.gen'
import type { DialogActions } from '@langgenius/dify-ui/dialog'
import type { DataSourceProvider, NotionPage } from '@/models/common'
import type { CrawlOptions, CrawlResultItem, FileItem } from '@/models/datasets'
import { cn } from '@langgenius/dify-ui/cn'
import { Dialog, DialogContent, DialogTrigger } from '@langgenius/dify-ui/dialog'
import { RiFolder6Line } from '@remixicon/react'
import { useQuery } from '@tanstack/react-query'
import { useBoolean } from 'ahooks'
import { useAtomValue } from 'jotai'
import { useCallback, useId, useMemo, useRef } from 'react'
import { useTranslation } from 'react-i18next'
import NotionConnector from '@/app/components/base/notion-connector'
import { NotionPageSelector } from '@/app/components/base/notion-page-selector'
import VectorSpaceFull from '@/app/components/billing/vector-space-full'
import VectorSpaceUnavailable from '@/app/components/billing/vector-space-unavailable'
import { toCreationDataSourceType } from '@/app/components/datasets/dataset-fields'
import { useDatasetDetailContextWithSelector } from '@/context/dataset-detail'
import { deploymentEditionAtom } from '@/features/system-features/state'
import { DataSourceType } from '@/models/datasets'
import { consoleQuery } from '@/service/console'
import EmptyDatasetCreationContent from '../empty-dataset-creation-modal'
import FileUploader from '../file-uploader'
import Website from '../website'
import { DataSourceTypeSelector, NextStepButton, PreviewPanel } from './components'
import { usePreviewState } from './hooks'
import s from './index.module.css'
import UpgradeCard from './upgrade-card'

type IStepOneProps = {
  datasetId?: string
  dataSourceType?: DataSourceType
  dataSourceTypeDisable: boolean
  onSetting: () => void
  files: FileItem[]
  updateFileList: (files: FileItem[]) => void
  updateFile: (fileItem: FileItem, progress: number, list: FileItem[]) => void
  notionPages?: NotionPage[]
  notionCredentialId: string
  updateNotionPages: (value: NotionPage[]) => void
  updateNotionCredentialId: (credentialId: string) => void
  onStepChange: () => void
  changeType: (type: DataSourceType) => void
  websitePages?: CrawlResultItem[]
  updateWebsitePages: (value: CrawlResultItem[]) => void
  onWebsiteCrawlProviderChange: (provider: DataSourceProvider) => void
  onWebsiteCrawlJobIdChange: (jobId: string) => void
  crawlOptions: CrawlOptions
  onCrawlOptionsChange: (payload: CrawlOptions) => void
  authedDataSourceList: DatasourceProviderAuthResponse[]
}

// Helper function to check if notion is authenticated
function checkNotionAuth(authedDataSourceList: DatasourceProviderAuthResponse[]): boolean {
  const notionSource = authedDataSourceList.find((item) => item.provider === 'notion_datasource')
  return Boolean(notionSource && notionSource.credentials_list.length > 0)
}

// Helper function to get notion credential list
function getNotionCredentialList(authedDataSourceList: DatasourceProviderAuthResponse[]) {
  return (
    authedDataSourceList.find((item) => item.provider === 'notion_datasource')?.credentials_list ||
    []
  )
}

// Lookup table for checking multiple items by data source type
const MULTIPLE_ITEMS_CHECK: Record<
  DataSourceType,
  (props: {
    files: FileItem[]
    notionPages: NotionPage[]
    websitePages: CrawlResultItem[]
  }) => boolean
> = {
  [DataSourceType.FILE]: ({ files }) => files.length > 1,
  [DataSourceType.NOTION]: ({ notionPages }) => notionPages.length > 1,
  [DataSourceType.WEB]: ({ websitePages }) => websitePages.length > 1,
}

const StepOne = ({
  datasetId,
  dataSourceType: inCreatePageDataSourceType,
  dataSourceTypeDisable,
  changeType,
  onSetting,
  onStepChange: doOnStepChange,
  files,
  updateFileList,
  updateFile,
  notionPages = [],
  notionCredentialId,
  updateNotionPages,
  updateNotionCredentialId,
  websitePages = [],
  updateWebsitePages,
  onWebsiteCrawlProviderChange,
  onWebsiteCrawlJobIdChange,
  crawlOptions,
  onCrawlOptionsChange,
  authedDataSourceList,
}: IStepOneProps) => {
  const { t } = useTranslation(['datasetCreation'])
  const sourceLabelId = useId()
  const dataset = useDatasetDetailContextWithSelector((state) => state.dataset)
  const deploymentEdition = useAtomValue(deploymentEditionAtom)
  const { data: plan } = useQuery(
    consoleQuery.features.get.queryOptions({
      enabled: deploymentEdition === 'CLOUD',
      select: (data) => data.billing.subscription.plan,
    }),
  )

  // Preview state management
  const {
    currentFile,
    currentNotionPage,
    currentWebsite,
    showFilePreview,
    hideFilePreview,
    showNotionPagePreview,
    hideNotionPagePreview,
    showWebsitePreview,
    hideWebsitePreview,
  } = usePreviewState()

  const emptyDatasetDialogActionsRef = useRef<DialogActions>(null)
  const isCreatingEmptyDatasetRef = useRef(false)

  // Plan upgrade modal state
  const [
    isShowPlanUpgradeModal,
    { setTrue: showPlanUpgradeModal, setFalse: hidePlanUpgradeModal },
  ] = useBoolean(false)

  // Computed values
  const shouldShowDataSourceTypeList = !datasetId || (datasetId && !dataset?.data_source_type)
  const isInCreatePage = shouldShowDataSourceTypeList
  // Default to FILE type when no type is provided from either source
  const dataSourceType = isInCreatePage
    ? (inCreatePageDataSourceType ?? DataSourceType.FILE)
    : (toCreationDataSourceType(dataset?.data_source_type) ?? DataSourceType.FILE)

  const allFileLoaded = files.length > 0 && files.every((file) => file.file.id)
  const hasNotion = notionPages.length > 0
  const shouldCheckVectorSpace = deploymentEdition === 'CLOUD' && (allFileLoaded || hasNotion)
  const {
    data: vectorSpace,
    isFetching: isFetchingVectorSpacePlan,
    refetch: refetchVectorSpace,
  } = useQuery(
    consoleQuery.features.vectorSpace.get.queryOptions({ enabled: shouldCheckVectorSpace }),
  )
  const isCheckingVectorSpace = shouldCheckVectorSpace && !vectorSpace && isFetchingVectorSpacePlan
  const isVectorSpaceUnavailable =
    shouldCheckVectorSpace && plan === 'sandbox' && !!vectorSpace?.usage_unknown
  const isVectorSpaceFull =
    !!vectorSpace &&
    !vectorSpace.usage_unknown &&
    vectorSpace.limit > 0 &&
    vectorSpace.size >= vectorSpace.limit
  const isShowVectorSpaceFull =
    (allFileLoaded || hasNotion) && isVectorSpaceFull && deploymentEdition === 'CLOUD'
  const isPlanUnavailable = deploymentEdition === 'CLOUD' && plan === undefined
  const supportBatchUpload =
    deploymentEdition !== 'CLOUD' || plan === 'professional' || plan === 'team'

  const isNotionAuthed = useMemo(
    () => checkNotionAuth(authedDataSourceList),
    [authedDataSourceList],
  )
  const notionCredentialList = useMemo(
    () => getNotionCredentialList(authedDataSourceList),
    [authedDataSourceList],
  )

  const fileNextDisabled = useMemo(() => {
    if (!files.length) return true
    if (files.some((file) => !file.file.id)) return true
    if (isCheckingVectorSpace) return true
    return isShowVectorSpaceFull || isVectorSpaceUnavailable
  }, [files, isCheckingVectorSpace, isShowVectorSpaceFull, isVectorSpaceUnavailable])

  // Clear previews when switching data source type
  const handleClearPreviews = useCallback(
    (newType: DataSourceType) => {
      if (newType !== DataSourceType.FILE) hideFilePreview()
      if (newType !== DataSourceType.NOTION) hideNotionPagePreview()
      if (newType !== DataSourceType.WEB) hideWebsitePreview()
    },
    [hideFilePreview, hideNotionPagePreview, hideWebsitePreview],
  )

  // Handle step change with batch upload check
  const onStepChange = useCallback(() => {
    if (isPlanUnavailable) return
    if (!supportBatchUpload && dataSourceType) {
      const checkFn = MULTIPLE_ITEMS_CHECK[dataSourceType]
      if (checkFn?.({ files, notionPages, websitePages })) {
        showPlanUpgradeModal()
        return
      }
    }
    doOnStepChange()
  }, [
    dataSourceType,
    doOnStepChange,
    files,
    isPlanUnavailable,
    supportBatchUpload,
    notionPages,
    showPlanUpgradeModal,
    websitePages,
  ])

  return (
    <div className="size-full min-w-0 overflow-y-auto">
      <div className="flex min-h-full w-full min-w-0 flex-col xl:h-full xl:flex-row">
        {/* Left Panel - Form */}
        <div className="relative w-full min-w-0 xl:h-full xl:w-1/2 xl:overflow-y-auto">
          <div className="flex min-w-0 justify-end">
            <div className={cn(s.form)}>
              {shouldShowDataSourceTypeList && (
                <>
                  <h1
                    id={sourceLabelId}
                    className={cn(s.stepHeader, 'system-md-semibold text-text-secondary')}
                  >
                    {t(($) => $['steps.one'], { ns: 'datasetCreation' })}
                  </h1>
                  <DataSourceTypeSelector
                    labelledBy={sourceLabelId}
                    currentType={dataSourceType}
                    disabled={dataSourceTypeDisable}
                    onChange={changeType}
                    onClearPreviews={handleClearPreviews}
                  />
                </>
              )}

              {/* File Data Source */}
              {dataSourceType === DataSourceType.FILE && (
                <>
                  <FileUploader
                    fileList={files}
                    titleClassName={
                      !shouldShowDataSourceTypeList ? 'mt-[30px] mb-[44px]! text-lg!' : undefined
                    }
                    prepareFileList={updateFileList}
                    onFileListUpdate={updateFileList}
                    onFileUpdate={updateFile}
                    onPreview={showFilePreview}
                    supportBatchUpload={supportBatchUpload}
                  />
                  {isShowVectorSpaceFull && (
                    <div className="mb-4 max-w-160">
                      <VectorSpaceFull />
                    </div>
                  )}
                  {isVectorSpaceUnavailable && (
                    <div className="mb-4 max-w-160">
                      <VectorSpaceUnavailable
                        isRetrying={isFetchingVectorSpacePlan}
                        onRetry={() => void refetchVectorSpace()}
                      />
                    </div>
                  )}
                  <NextStepButton
                    disabled={isPlanUnavailable || fileNextDisabled}
                    onClick={onStepChange}
                  />
                  {deploymentEdition === 'CLOUD' && plan === 'sandbox' && (
                    <div className="mt-5">
                      <div className="mb-4 h-px bg-divider-subtle" />
                      <UpgradeCard />
                    </div>
                  )}
                </>
              )}

              {/* Notion Data Source */}
              {dataSourceType === DataSourceType.NOTION && (
                <>
                  {!isNotionAuthed && (
                    <div
                      className={cn(
                        'mb-8 w-full max-w-160',
                        !shouldShowDataSourceTypeList && 'mt-12',
                      )}
                    >
                      <NotionConnector onSetting={onSetting} />
                    </div>
                  )}
                  {isNotionAuthed && (
                    <>
                      <div className="mb-8 w-full max-w-160">
                        <NotionPageSelector
                          value={notionPages.map((page) => page.page_id)}
                          onSelect={updateNotionPages}
                          onPreview={showNotionPagePreview}
                          credentialList={notionCredentialList}
                          onSelectCredential={updateNotionCredentialId}
                          datasetId={datasetId}
                        />
                      </div>
                      {isShowVectorSpaceFull && (
                        <div className="mb-4 max-w-160">
                          <VectorSpaceFull />
                        </div>
                      )}
                      {isVectorSpaceUnavailable && (
                        <div className="mb-4 max-w-160">
                          <VectorSpaceUnavailable
                            isRetrying={isFetchingVectorSpacePlan}
                            onRetry={() => void refetchVectorSpace()}
                          />
                        </div>
                      )}
                      <NextStepButton
                        disabled={
                          isPlanUnavailable ||
                          isShowVectorSpaceFull ||
                          isVectorSpaceUnavailable ||
                          !notionPages.length
                        }
                        onClick={onStepChange}
                      />
                    </>
                  )}
                </>
              )}

              {/* Web Data Source */}
              {dataSourceType === DataSourceType.WEB && (
                <>
                  <div
                    className={cn(
                      'mb-8 w-full max-w-160',
                      !shouldShowDataSourceTypeList && 'mt-12',
                    )}
                  >
                    <Website
                      onPreview={showWebsitePreview}
                      checkedCrawlResult={websitePages}
                      onCheckedCrawlResultChange={updateWebsitePages}
                      onCrawlProviderChange={onWebsiteCrawlProviderChange}
                      onJobIdChange={onWebsiteCrawlJobIdChange}
                      crawlOptions={crawlOptions}
                      onCrawlOptionsChange={onCrawlOptionsChange}
                      authedDataSourceList={authedDataSourceList}
                    />
                  </div>
                  {isShowVectorSpaceFull && (
                    <div className="mb-4 max-w-160">
                      <VectorSpaceFull />
                    </div>
                  )}
                  <NextStepButton
                    disabled={isPlanUnavailable || isShowVectorSpaceFull || !websitePages.length}
                    onClick={onStepChange}
                  />
                </>
              )}

              {/* Empty Dataset Creation Link */}
              {!datasetId && (
                <>
                  <div className="my-8 h-px max-w-160 bg-divider-regular" />
                  <Dialog
                    actionsRef={emptyDatasetDialogActionsRef}
                    onOpenChange={(open, details) => {
                      if (
                        !open &&
                        isCreatingEmptyDatasetRef.current &&
                        details.reason !== 'imperative-action'
                      )
                        details.cancel()
                    }}
                  >
                    <DialogTrigger
                      type="button"
                      className="inline-flex cursor-pointer items-center rounded-sm text-start text-[13px] leading-4 text-text-accent focus-visible:outline-2 focus-visible:outline-offset-2 focus-visible:outline-components-input-border-active"
                    >
                      <RiFolder6Line aria-hidden="true" className="me-1 size-4 shrink-0" />
                      {t(($) => $['stepOne.emptyDatasetCreation'], { ns: 'datasetCreation' })}
                    </DialogTrigger>
                    <DialogContent className="w-130">
                      <EmptyDatasetCreationContent
                        onCreated={() => emptyDatasetDialogActionsRef.current?.close()}
                        onPendingChange={(pending) => {
                          isCreatingEmptyDatasetRef.current = pending
                        }}
                      />
                    </DialogContent>
                  </Dialog>
                </>
              )}
            </div>
          </div>
        </div>

        {/* Right Panel - Preview */}
        <PreviewPanel
          currentFile={currentFile}
          currentNotionPage={currentNotionPage}
          currentWebsite={currentWebsite}
          notionCredentialId={notionCredentialId}
          isShowPlanUpgradeModal={isShowPlanUpgradeModal}
          hideFilePreview={() => hideFilePreview(true)}
          hideNotionPagePreview={hideNotionPagePreview}
          hideWebsitePreview={hideWebsitePreview}
          hidePlanUpgradeModal={hidePlanUpgradeModal}
        />
      </div>
    </div>
  )
}

export default StepOne