Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

2.9 KB
Raw Blame
"""Gated poller for KnowledgeFS lifecycle outbox delivery and reconciliation."""

from __future__ import annotations

import socket
from datetime import timedelta
from typing import TypedDict

from celery import shared_task

from configs import dify_config
from core.db.session_factory import session_factory
from libs.datetime_utils import naive_utc_now
from services.knowledge_fs.lifecycle_readiness import get_configured_knowledge_fs_lifecycle_worker_readiness
from services.knowledge_fs.lifecycle_saga import KnowledgeFSLifecycleSagaRunner
from services.knowledge_fs.orphan_reconciler import KnowledgeFSOrphanReconciler
from services.knowledge_fs.remote_registry import get_knowledge_fs_lifecycle_remote
from services.knowledge_fs.runtime import get_knowledge_fs_runtime
from services.knowledge_fs.staged_upload_service import KnowledgeFSStagedUploadService


class KnowledgeFSLifecycleWorkerResult(TypedDict):
    status: str
    dispatched: int
    completed: int
    reconciled: int


@shared_task(queue="knowledge_fs_lifecycle")
def run_knowledge_fs_lifecycle_worker() -> KnowledgeFSLifecycleWorkerResult:
    readiness = get_configured_knowledge_fs_lifecycle_worker_readiness()
    if not readiness.ready:
        return {"status": "disabled", "dispatched": 0, "completed": 0, "reconciled": 0}

    remote = get_knowledge_fs_lifecycle_remote()
    session_maker = session_factory.get_session_maker()
    runner = KnowledgeFSLifecycleSagaRunner(session_maker, remote)
    worker_id = f"{socket.gethostname()}:{run_knowledge_fs_lifecycle_worker.request.id or 'manual'}"
    dispatched = 0
    completed = 0
    for _ in range(dify_config.KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE):
        result = runner.dispatch_one(
            worker_id=worker_id,
            now=naive_utc_now(),
            lease_duration=timedelta(seconds=dify_config.KNOWLEDGE_FS_LIFECYCLE_LEASE_SECONDS),
            product_enabled=dify_config.KNOWLEDGE_FS_ENABLED,
        )
        if not result.claimed:
            break
        dispatched += 1
        completed += result.completed
    report = KnowledgeFSOrphanReconciler(session_maker, remote).reconcile(
        limit=dify_config.KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE,
        apply_repairs=True,
    )
    reconciled = len(report.repaired_control_space_ids) + len(report.cleanup_control_space_ids)
    return {"status": "ok", "dispatched": dispatched, "completed": completed, "reconciled": reconciled}


@shared_task(queue="knowledge_fs_lifecycle")
def cleanup_knowledge_fs_staged_uploads() -> int:
    readiness = get_configured_knowledge_fs_lifecycle_worker_readiness()
    if not readiness.ready:
        return 0
    session_maker = session_factory.get_session_maker()
    runtime = get_knowledge_fs_runtime(session_maker)
    return KnowledgeFSStagedUploadService(session_maker, facade=runtime.facade).cleanup_expired(
        limit=dify_config.KNOWLEDGE_FS_LIFECYCLE_BATCH_SIZE
    )


__all__ = ["cleanup_knowledge_fs_staged_uploads", "run_knowledge_fs_lifecycle_worker"]