Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

11.9 KB
Raw Blame
import logging
import time
from collections.abc import Sequence
from typing import Any, Protocol

import click
from celery import current_app, shared_task
from sqlalchemy import select

from configs import dify_config
from core.db.session_factory import session_factory
from core.entities.document_task import DocumentTask
from core.rag.index_processor.constant.index_type import IndexStructureType, IndexTechniqueType
from core.rag.pipeline.queue import TenantIsolatedTaskQueue
from enums import CloudPlan, DeploymentEdition
from libs.datetime_utils import naive_utc_now
from models.dataset import Dataset, Document
from models.enums import IndexingStatus
from services.feature_service import FeatureService
from services.knowledge.indexing.adapters.execution import build_document_indexing_service
from services.knowledge.indexing.errors import DocumentIsPausedError
from services.knowledge.resource_scope import DatasetRef
from tasks.generate_summary_index_task import generate_summary_index_task

logger = logging.getLogger(__name__)


class CeleryTaskLike(Protocol):
    def delay(self, *args: Any, **kwargs: Any) -> Any: ...

    def apply_async(self, *args: Any, **kwargs: Any) -> Any: ...


@shared_task(queue="dataset")
def document_indexing_task(dataset_id: str, document_ids: list):
    """
    Async process document
    :param dataset_id:
    :param document_ids:

    .. warning:: TO BE DEPRECATED
        This function will be deprecated and removed in a future version.
        Use normal_document_indexing_task or priority_document_indexing_task instead.

    Usage: document_indexing_task.delay(dataset_id, document_ids)
    """
    logger.warning("document indexing legacy mode received: %s - %s", dataset_id, document_ids)
    _document_indexing(dataset_id, document_ids)


def _document_indexing(dataset_id: str, document_ids: Sequence[str]):
    """
    Process document for tasks
    :param dataset_id:
    :param document_ids:

    Usage: _document_indexing(dataset_id, document_ids)
    """
    start_at = time.perf_counter()

    with session_factory.create_session() as session:
        dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id).limit(1))
        if not dataset:
            logger.info(click.style(f"Dataset is not found: {dataset_id}", fg="yellow"))
            return
        # check document limit
        if dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD:
            features = FeatureService.get_features(dataset.tenant_id)
            try:
                vector_space = features.vector_space
                assert vector_space is not None
                count = len(document_ids)
                batch_upload_limit = int(dify_config.BATCH_UPLOAD_LIMIT)
                if features.billing.subscription.plan == CloudPlan.SANDBOX and count > 1:
                    raise ValueError("Your current plan does not support batch upload, please upgrade your plan.")
                if count > batch_upload_limit:
                    raise ValueError(f"You have reached the batch upload limit of {batch_upload_limit}.")
                if 0 < vector_space.limit <= vector_space.size:
                    raise ValueError(
                        "Your total number of documents plus the number of uploads have over the limit of "
                        "your subscription."
                    )
            except Exception as e:
                for document_id in document_ids:
                    document = session.scalar(
                        select(Document).where(Document.id == document_id, Document.dataset_id == dataset_id).limit(1)
                    )
                    if document:
                        document.indexing_status = IndexingStatus.ERROR
                        document.error = str(e)
                        document.stopped_at = naive_utc_now()
                        session.add(document)
                session.commit()
                return

    # Phase 1: Persist parsing status before slow extraction and vector operations.
    with session_factory.create_session() as session, session.begin():
        documents: list[Document] = list(
            session.scalars(
                select(Document).where(Document.id.in_(document_ids), Document.dataset_id == dataset_id)
            ).all()
        )

        for document in documents:
            document.indexing_status = IndexingStatus.PARSING
            document.processing_started_at = naive_utc_now()
            session.add(document)

    # Phase 2: Execute indexing without holding locks from the parsing-status update.
    has_error = False
    try:
        indexing_service = build_document_indexing_service(
            session_factory=session_factory.get_session_maker(), enforce_vector_space_admission=True
        )
        with session_factory.create_session() as session:
            dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id).limit(1))
            if not dataset:
                logger.info(click.style(f"Dataset is not found: {dataset_id}", fg="yellow"))
                return

            documents = list(
                session.scalars(
                    select(Document).where(Document.id.in_(document_ids), Document.dataset_id == dataset_id)
                ).all()
            )

            document_refs = [DatasetRef(doc.tenant_id, doc.dataset_id).document(doc.id) for doc in documents]
            session.commit()
            indexing_service.run(document_refs)

        end_at = time.perf_counter()
        logger.info(click.style(f"Processed dataset: {dataset_id} latency: {end_at - start_at}", fg="green"))
    except DocumentIsPausedError as ex:
        logger.info(click.style(str(ex), fg="yellow"))
        has_error = True
    except Exception:
        logger.exception("Document indexing task failed, dataset_id: %s", dataset_id)
        has_error = True

    if not has_error:
        with session_factory.create_session() as session:
            dataset = session.scalar(select(Dataset).where(Dataset.id == dataset_id).limit(1))
            if not dataset:
                logger.warning("Dataset %s not found after indexing", dataset_id)
                return

            if dataset.indexing_technique == IndexTechniqueType.HIGH_QUALITY:
                summary_index_setting = dataset.summary_index_setting
                if summary_index_setting and summary_index_setting.get("enable"):
                    # Check each document's indexing status and trigger summary generation if completed

                    documents = list(
                        session.scalars(
                            select(Document).where(Document.id.in_(document_ids), Document.dataset_id == dataset_id)
                        ).all()
                    )

                    for document in documents:
                        if document:
                            logger.info(
                                "Checking document %s for summary generation: status=%s, doc_form=%s, need_summary=%s",
                                document.id,
                                document.indexing_status,
                                document.doc_form,
                                document.need_summary,
                            )
                            if (
                                document.indexing_status == IndexingStatus.COMPLETED
                                and document.doc_form != IndexStructureType.QA_INDEX
                                and document.need_summary is True
                            ):
                                try:
                                    generate_summary_index_task.delay(dataset.id, document.id, None)
                                    logger.info(
                                        "Queued summary index generation task for document %s in dataset %s "
                                        "after indexing completed",
                                        document.id,
                                        dataset.id,
                                    )
                                except Exception:
                                    logger.exception(
                                        "Failed to queue summary index generation task for document %s",
                                        document.id,
                                    )
                                    # Don't fail the entire indexing process if summary task queuing fails
                            else:
                                logger.info(
                                    "Skipping summary generation for document %s: "
                                    "status=%s, doc_form=%s, need_summary=%s",
                                    document.id,
                                    document.indexing_status,
                                    document.doc_form,
                                    document.need_summary,
                                )
                        else:
                            logger.warning("Document %s not found after indexing", document.id)
            else:
                logger.info(
                    "Summary index generation skipped for dataset %s: indexing_technique=%s (not 'high_quality')",
                    dataset.id,
                    dataset.indexing_technique,
                )


def _document_indexing_with_tenant_queue(
    tenant_id: str, dataset_id: str, document_ids: Sequence[str], task_func: CeleryTaskLike
) -> None:
    try:
        _document_indexing(dataset_id, document_ids)
    except Exception:
        logger.exception(
            "Error processing document indexing %s for tenant %s: %s",
            dataset_id,
            tenant_id,
            document_ids,
            exc_info=True,
        )
    finally:
        tenant_isolated_task_queue = TenantIsolatedTaskQueue(tenant_id, "document_indexing")

        # Check if there are waiting tasks in the queue
        # Use rpop to get the next task from the queue (FIFO order)
        next_tasks = tenant_isolated_task_queue.pull_tasks(count=dify_config.TENANT_ISOLATED_TASK_CONCURRENCY)

        logger.info("document indexing tenant isolation queue %s next tasks: %s", tenant_id, next_tasks)

        if next_tasks:
            with current_app.producer_or_acquire() as producer:  # type: ignore
                for next_task in next_tasks:
                    document_task = DocumentTask(**next_task)
                    # Keep the flag set to indicate a task is running
                    tenant_isolated_task_queue.set_task_waiting_time()
                    task_func.apply_async(
                        kwargs={
                            "tenant_id": document_task.tenant_id,
                            "dataset_id": document_task.dataset_id,
                            "document_ids": document_task.document_ids,
                        },
                        producer=producer,
                    )

        else:
            # No more waiting tasks, clear the flag
            tenant_isolated_task_queue.delete_task_key()


@shared_task(queue="dataset")
def normal_document_indexing_task(tenant_id: str, dataset_id: str, document_ids: Sequence[str]):
    """
    Async process document
    :param tenant_id:
    :param dataset_id:
    :param document_ids:

    Usage: normal_document_indexing_task.delay(tenant_id, dataset_id, document_ids)
    """
    logger.info("normal document indexing task received: %s - %s - %s", tenant_id, dataset_id, document_ids)
    _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, normal_document_indexing_task)


@shared_task(queue="priority_dataset")
def priority_document_indexing_task(tenant_id: str, dataset_id: str, document_ids: Sequence[str]):
    """
    Priority async process document
    :param tenant_id:
    :param dataset_id:
    :param document_ids:

    Usage: priority_document_indexing_task.delay(tenant_id, dataset_id, document_ids)
    """
    logger.info("priority document indexing task received: %s - %s - %s", tenant_id, dataset_id, document_ids)
    _document_indexing_with_tenant_queue(tenant_id, dataset_id, document_ids, priority_document_indexing_task)