Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

7.6 KB
Raw Blame
import datetime
import time
from typing import TypedDict

import click
from sqlalchemy import func, select, update
from sqlalchemy.exc import SQLAlchemyError

import app
from configs import dify_config
from core.db.session_factory import session_factory
from core.rag.index_processor.index_processor import IndexProcessorFactory
from enums import CloudPlan
from extensions.ext_redis import redis_client
from libs.pagination import paginate_query
from models.dataset import Dataset, DatasetAutoDisableLog, DatasetQuery, Document
from repositories.knowledge.dataset_read_repository import get_dataset_doc_form
from services.feature_service import FeatureService


class CleanupConfig(TypedDict):
    clean_day: datetime.datetime
    plan_filter: str | None
    add_logs: bool


@app.celery.task(queue="dataset")
def clean_unused_datasets_task():
    click.echo(click.style("Start clean unused datasets indexes.", fg="green"))
    start_at = time.perf_counter()

    # Define cleanup configurations
    cleanup_configs: list[CleanupConfig] = [
        {
            "clean_day": datetime.datetime.now() - datetime.timedelta(days=dify_config.PLAN_SANDBOX_CLEAN_DAY_SETTING),
            "plan_filter": None,
            "add_logs": True,
        },
        {
            "clean_day": datetime.datetime.now() - datetime.timedelta(days=dify_config.PLAN_PRO_CLEAN_DAY_SETTING),
            "plan_filter": CloudPlan.SANDBOX,
            "add_logs": False,
        },
    ]

    for config in cleanup_configs:
        clean_day = config["clean_day"]
        plan_filter = config["plan_filter"]
        add_logs = config["add_logs"]

        page = 1
        while True:
            try:
                # Subquery for counting new documents
                document_subquery_new = (
                    select(Document.dataset_id, func.count(Document.id).label("document_count"))
                    .where(
                        Document.indexing_status == "completed",
                        Document.enabled.is_(True),
                        Document.archived.is_(False),
                        Document.updated_at > clean_day,
                    )
                    .group_by(Document.dataset_id)
                    .subquery()
                )

                # Subquery for counting old documents
                document_subquery_old = (
                    select(Document.dataset_id, func.count(Document.id).label("document_count"))
                    .where(
                        Document.indexing_status == "completed",
                        Document.enabled.is_(True),
                        Document.archived.is_(False),
                        Document.updated_at < clean_day,
                    )
                    .group_by(Document.dataset_id)
                    .subquery()
                )

                # Main query with join and filter
                stmt = (
                    select(Dataset)
                    .outerjoin(document_subquery_new, Dataset.id == document_subquery_new.c.dataset_id)
                    .outerjoin(document_subquery_old, Dataset.id == document_subquery_old.c.dataset_id)
                    .where(
                        Dataset.created_at < clean_day,
                        func.coalesce(document_subquery_new.c.document_count, 0) == 0,
                        func.coalesce(document_subquery_old.c.document_count, 0) > 0,
                    )
                    .order_by(Dataset.created_at.desc())
                )

                with session_factory.create_session() as session:
                    datasets = paginate_query(stmt, page=page, per_page=50, session=session)

                    if datasets is None or datasets.items is None or len(datasets.items) == 0:
                        break

                    for dataset in datasets:
                        dataset_query = session.scalars(
                            select(DatasetQuery).where(
                                DatasetQuery.created_at > clean_day, DatasetQuery.dataset_id == dataset.id
                            )
                        ).all()

                        if not dataset_query or len(dataset_query) == 0:
                            try:
                                should_clean = True

                                # Check plan filter if specified
                                if plan_filter:
                                    features_cache_key = f"features:{dataset.tenant_id}"
                                    plan_cache = redis_client.get(features_cache_key)
                                    if plan_cache is None:
                                        features = FeatureService.get_features(
                                            dataset.tenant_id, exclude_vector_space=True
                                        )
                                        redis_client.setex(features_cache_key, 600, features.billing.subscription.plan)
                                        plan = features.billing.subscription.plan
                                    else:
                                        plan = plan_cache.decode()
                                    should_clean = plan == plan_filter

                                if should_clean:
                                    # Add auto disable log if required
                                    if add_logs:
                                        documents = session.scalars(
                                            select(Document).where(
                                                Document.dataset_id == dataset.id,
                                                Document.enabled.is_(True),
                                                Document.archived.is_(False),
                                            )
                                        ).all()
                                        for document in documents:
                                            dataset_auto_disable_log = DatasetAutoDisableLog(
                                                tenant_id=dataset.tenant_id,
                                                dataset_id=dataset.id,
                                                document_id=document.id,
                                            )
                                            session.add(dataset_auto_disable_log)

                                    # Remove index
                                    index_processor = IndexProcessorFactory(
                                        get_dataset_doc_form(dataset, session=session)
                                    ).init_index_processor()
                                    index_processor.clean(dataset, None, session=session)

                                    # Update document
                                    session.execute(
                                        update(Document).where(Document.dataset_id == dataset.id).values(enabled=False)
                                    )
                                    session.commit()
                                    click.echo(
                                        click.style(f"Cleaned unused dataset {dataset.id} from db success!", fg="green")
                                    )
                            except Exception as e:
                                session.rollback()
                                click.echo(
                                    click.style(f"clean dataset index error: {e.__class__.__name__} {str(e)}", fg="red")
                                )

            except SQLAlchemyError:
                raise

            page += 1

    end_at = time.perf_counter()
    click.echo(click.style(f"Cleaned unused dataset from db success latency: {end_at - start_at}", fg="green"))