Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

4.1 KB
Raw Blame
"""Celery entry points of the trace pipeline, routed by ``OPS_TRACE_QUEUE``.

Delivery is at most once per sink: an event is redelivered only while its
records are not persisted yet, or when the Agent collector could not decide
ownership (enterprise telemetry is then skipped, it already ran).
"""

import logging
import re
from datetime import UTC, date, datetime, timedelta
from typing import Any

from celery import shared_task
from dify_trace_sdk.events import TraceEvent

from extensions.ext_application_services import application_services
from extensions.ext_storage import storage
from services.tracing.adapters import PAYLOAD_PREFIX
from services.tracing.delivery import ExportJob
from services.tracing.errors import SourceNotFoundError, SourceNotReadyError, TraceCollectorUnavailableError

logger = logging.getLogger(__name__)

REDELIVERY_LIMIT = 10
PAYLOAD_RETENTION = timedelta(days=2)
_PAYLOAD_DAY = re.compile(rf"{PAYLOAD_PREFIX}/(\d{{4}}-\d{{2}}-\d{{2}})/")


def _expired(key: str, cutoff: date) -> bool:
    match = _PAYLOAD_DAY.match(key)
    if match is None:
        return False
    try:
        return date.fromisoformat(match.group(1)) < cutoff
    except ValueError:
        return False


def _redelivery_delay(retries: int) -> int:
    return min(60, 5 * 2**retries)


def _delete_quietly(key: str) -> None:
    try:
        storage.delete(key)
    except Exception:
        logger.warning("Trace payload cleanup failed: key=%s", key, exc_info=True)


@shared_task(bind=True, max_retries=REDELIVERY_LIMIT)
def deliver_trace_event(self, envelope: dict[str, Any]) -> None:
    payload_key = envelope.get("payload_key")
    redelivering = False
    try:
        raw = storage.load(payload_key) if isinstance(payload_key, str) else envelope["event"]
        event = TraceEvent.model_validate_json(raw)
    except Exception:
        logger.warning("Trace event unreadable; dropped", exc_info=True)
        if isinstance(payload_key, str):
            _delete_quietly(payload_key)
        return
    try:
        application_services().tracing.delivery.deliver(
            event, enterprise_delivered=envelope.get("enterprise_delivered") is True
        )
    except SourceNotFoundError:
        logger.info("Trace event source not found; dropped: kind=%s app_id=%s", event.payload.kind, event.app_id)
    except SourceNotReadyError:
        if self.request.retries < REDELIVERY_LIMIT:
            redelivering = True
            raise self.retry(args=(envelope,), countdown=_redelivery_delay(self.request.retries)) from None
        logger.warning("Trace event source never became ready; dropped: kind=%s", event.payload.kind)
    except TraceCollectorUnavailableError:
        if self.request.retries < REDELIVERY_LIMIT:
            redelivering = True
            raise self.retry(
                args=({**envelope, "enterprise_delivered": True},),
                countdown=_redelivery_delay(self.request.retries),
            ) from None
        logger.warning("Agent trace collector unavailable; dropped: kind=%s", event.payload.kind)
    finally:
        if isinstance(payload_key, str) and not redelivering:
            _delete_quietly(payload_key)


@shared_task
def retry_trace_export(job_key: str) -> None:
    try:
        job = ExportJob.model_validate_json(storage.load(job_key))
    except Exception:
        logger.warning("Trace export job unreadable; dropped: key=%s", job_key, exc_info=True)
        _delete_quietly(job_key)
        return
    try:
        application_services().tracing.delivery.retry(job)
    finally:
        _delete_quietly(job_key)


@shared_task
def sweep_trace_payloads() -> None:
    """Delete expired new-pipeline payloads; legacy workers still own their payloads during an upgrade."""
    cutoff = (datetime.now(UTC) - PAYLOAD_RETENTION).date()
    # Directory listings need the trailing slash on some storage backends.
    prefix = f"{PAYLOAD_PREFIX}/"
    try:
        keys = storage.scan(prefix, files=True)
    except (FileNotFoundError, NotImplementedError):
        return
    for key in keys:
        if key.startswith(prefix) and _expired(key, cutoff):
            _delete_quietly(key)