Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

17.4 KB
Raw Blame
"""Tenant credit pool accounting.

Credit deductions are guarded by a tenant-level Redis lock before the database
row lock is acquired. This keeps concurrent usage accounting for one tenant
from piling up database transactions while preserving cross-tenant concurrency.
"""

import logging
from collections.abc import Callable, Mapping
from dataclasses import dataclass, field
from enum import StrEnum, auto
from typing import Any
from uuid import uuid4

from sqlalchemy import select
from sqlalchemy.orm import Session

from configs import dify_config
from core.credit_usage import normalize_credit_usage_app_type, normalize_credit_usage_created_by
from core.errors.error import QuotaExceededError
from enums import DeploymentEdition
from extensions.ext_redis import redis_client
from models import TenantCreditPool
from models.enums import ProviderQuotaType

logger = logging.getLogger(__name__)

FEATURE_KEY_CREDIT_POOL = "credit_pool"
CREDIT_USAGE_CREATED_BY_META_KEY = "created_by"
CREDIT_USAGE_APP_TYPE_META_KEY = "app_type"
CREDIT_POOL_TENANT_LOCK_TIMEOUT_SECONDS = 10
CREDIT_POOL_TENANT_LOCK_BLOCKING_TIMEOUT_SECONDS = 5


@dataclass(frozen=True)
class CreditPoolBalance:
    tenant_id: str
    pool_type: str
    quota_limit: int
    quota_used: int
    exhausted_at: int | None = None

    @property
    def remaining_credits(self) -> int:
        if self.quota_limit == -1:
            return -1
        return max(0, self.quota_limit - self.quota_used)

    def has_sufficient_credits(self, required_credits: int) -> bool:
        return self.quota_limit == -1 or self.remaining_credits >= required_credits


class CreditPoolReservationState(StrEnum):
    RESERVED = auto()
    COMMITTED = auto()
    RELEASED = auto()


@dataclass
class CreditPoolReservation:
    """A strict credit-pool reservation spanning one billable operation."""

    tenant_id: str
    pool_type: str
    amount: int
    request_id: str
    reservation_id: str | None
    meta: dict[str, Any] = field(default_factory=dict)
    _session_factory: Callable[[], Session] | None = field(default=None, repr=False)
    _state: CreditPoolReservationState = field(default=CreditPoolReservationState.RESERVED, init=False, repr=False)

    @property
    def state(self) -> CreditPoolReservationState:
        return self._state

    def commit(self) -> None:
        if self._state == CreditPoolReservationState.COMMITTED:
            return
        if self._state == CreditPoolReservationState.RELEASED:
            raise RuntimeError("Cannot commit a released credit reservation.")

        if self.reservation_id is not None:
            from services.billing_service import BillingService

            BillingService.quota_commit(
                tenant_id=self.tenant_id,
                feature_key=FEATURE_KEY_CREDIT_POOL,
                bucket=self.pool_type,
                reservation_id=self.reservation_id,
                actual_amount=self.amount,
                meta={**self.meta, "request_id": self.request_id},
            )

        # The database fallback reserves by deducting under the tenant lock, so
        # commit only makes that already durable reservation final.
        self._state = CreditPoolReservationState.COMMITTED

    def release(self) -> None:
        if self._state in {CreditPoolReservationState.COMMITTED, CreditPoolReservationState.RELEASED}:
            return

        if self.reservation_id is not None:
            from services.billing_service import BillingService

            BillingService.quota_release(
                tenant_id=self.tenant_id,
                feature_key=FEATURE_KEY_CREDIT_POOL,
                bucket=self.pool_type,
                reservation_id=self.reservation_id,
            )
        else:
            if self._session_factory is None:
                raise RuntimeError("Database credit reservation requires a session factory.")
            CreditPoolService._release_database_reservation(
                tenant_id=self.tenant_id,
                pool_type=self.pool_type,
                credits=self.amount,
                session=self._session_factory(),
            )

        self._state = CreditPoolReservationState.RELEASED


class CreditPoolService:
    @staticmethod
    def _normalize_pool_type(pool_type: str | ProviderQuotaType) -> str:
        return pool_type.value if isinstance(pool_type, ProviderQuotaType) else str(pool_type)

    @staticmethod
    def _build_billing_metadata(
        source: str,
        metadata: Mapping[str, object] | None,
    ) -> dict[str, object]:
        billing_metadata: dict[str, object] = {
            "source": source,
            **dict(metadata or {}),
        }
        billing_metadata[CREDIT_USAGE_CREATED_BY_META_KEY] = normalize_credit_usage_created_by(
            billing_metadata.get(CREDIT_USAGE_CREATED_BY_META_KEY)
        ).value
        billing_metadata[CREDIT_USAGE_APP_TYPE_META_KEY] = normalize_credit_usage_app_type(
            billing_metadata.get(CREDIT_USAGE_APP_TYPE_META_KEY)
        ).value
        return billing_metadata

    @staticmethod
    def _use_billing_quota() -> bool:
        return bool(dify_config.DEPLOYMENT_EDITION == DeploymentEdition.CLOUD)

    @staticmethod
    def _require_session(session: Session | None) -> Session:
        if session is None:
            raise ValueError("session is required when billing quota is disabled")
        return session

    @staticmethod
    def _get_tenant_lock_key(tenant_id: str) -> str:
        return f"credit_pool:tenant:{tenant_id}:deduct_lock"

    @classmethod
    def _deduct_with_tenant_lock(cls, tenant_id: str, deduct: Callable[[], int]) -> int:
        lock_key = cls._get_tenant_lock_key(tenant_id)
        lock = redis_client.lock(
            lock_key,
            timeout=CREDIT_POOL_TENANT_LOCK_TIMEOUT_SECONDS,
            blocking_timeout=CREDIT_POOL_TENANT_LOCK_BLOCKING_TIMEOUT_SECONDS,
        )
        lock_acquired = False

        try:
            lock_acquired = lock.acquire(blocking=True)
            if not lock_acquired:
                raise QuotaExceededError("Failed to acquire credit pool lock")

            return deduct()
        finally:
            if lock_acquired:
                try:
                    lock.release()
                except Exception:
                    logger.warning("Failed to release credit pool lock, tenant_id=%s", tenant_id, exc_info=True)

    @staticmethod
    def _get_locked_pool(session: Session, tenant_id: str, pool_type: str) -> TenantCreditPool | None:
        return session.scalar(
            select(TenantCreditPool)
            .where(
                TenantCreditPool.tenant_id == tenant_id,
                TenantCreditPool.pool_type == pool_type,
            )
            .limit(1)
            .with_for_update()
        )

    @classmethod
    def get_pool(
        cls,
        tenant_id: str,
        pool_type: str | ProviderQuotaType = "trial",
        *,
        session: Session | None = None,
    ) -> TenantCreditPool | CreditPoolBalance | None:
        """get tenant credit pool"""
        normalized_pool_type = cls._normalize_pool_type(pool_type)
        if cls._use_billing_quota():
            from services.billing_service import BillingService

            balance = BillingService.quota_get_balance(
                tenant_id=tenant_id,
                feature_key=FEATURE_KEY_CREDIT_POOL,
                bucket=normalized_pool_type,
            )
            return CreditPoolBalance(
                tenant_id=tenant_id,
                pool_type=normalized_pool_type,
                quota_limit=balance.quota,
                quota_used=balance.usage,
                exhausted_at=balance.exhausted_at,
            )

        session = cls._require_session(session)
        return session.scalar(
            select(TenantCreditPool)
            .where(
                TenantCreditPool.tenant_id == tenant_id,
                TenantCreditPool.pool_type == normalized_pool_type,
            )
            .limit(1)
        )

    @classmethod
    def check_credits_available(
        cls,
        tenant_id: str,
        credits_required: int,
        pool_type: str | ProviderQuotaType = "trial",
        *,
        session: Session | None = None,
    ) -> bool:
        """check if credits are available without deducting"""
        pool = cls.get_pool(tenant_id, pool_type, session=session)
        if not pool:
            return False
        return pool.has_sufficient_credits(credits_required)

    @classmethod
    def reserve_credits(
        cls,
        tenant_id: str,
        credits_required: int,
        pool_type: str | ProviderQuotaType = "trial",
        *,
        request_id: str,
        session_factory: Callable[[], Session] | None = None,
        meta: dict[str, Any] | None = None,
    ) -> CreditPoolReservation:
        """Reserve the full amount or raise before the billable operation starts."""
        if credits_required <= 0:
            raise ValueError("credits_required must be greater than 0")
        if not request_id:
            raise ValueError("request_id is required")

        normalized_pool_type = cls._normalize_pool_type(pool_type)
        reservation_meta = cls._build_billing_metadata("credit_pool.reservation", meta)
        if cls._use_billing_quota():
            from services.billing_service import BillingService

            result = BillingService.quota_reserve(
                tenant_id=tenant_id,
                feature_key=FEATURE_KEY_CREDIT_POOL,
                bucket=normalized_pool_type,
                request_id=request_id,
                amount=credits_required,
                meta=reservation_meta,
            )
            reservation_id = result.reservation_id
            if not reservation_id:
                raise QuotaExceededError("Insufficient credits remaining")
            return CreditPoolReservation(
                tenant_id=tenant_id,
                pool_type=normalized_pool_type,
                amount=credits_required,
                request_id=request_id,
                reservation_id=reservation_id,
                meta=reservation_meta,
            )

        if session_factory is None:
            raise ValueError("session_factory is required when billing quota is disabled")

        session = session_factory()

        def reserve() -> int:
            pool = cls._get_locked_pool(session=session, tenant_id=tenant_id, pool_type=normalized_pool_type)
            if not pool:
                raise QuotaExceededError("Credit pool not found")
            if not pool.has_sufficient_credits(credits_required):
                raise QuotaExceededError("Insufficient credits remaining")

            pool.quota_used += credits_required
            session.commit()
            return credits_required

        try:
            cls._deduct_with_tenant_lock(tenant_id, reserve)
        except QuotaExceededError:
            session.rollback()
            raise
        except Exception as caught_error:
            session.rollback()
            logger.exception("Failed to reserve credits for tenant %s", tenant_id)
            raise QuotaExceededError("Failed to reserve credits") from caught_error

        return CreditPoolReservation(
            tenant_id=tenant_id,
            pool_type=normalized_pool_type,
            amount=credits_required,
            request_id=request_id,
            reservation_id=None,
            meta=reservation_meta,
            _session_factory=session_factory,
        )

    @classmethod
    def _release_database_reservation(
        cls,
        *,
        tenant_id: str,
        pool_type: str,
        credits: int,
        session: Session,
    ) -> None:
        def release() -> int:
            pool = cls._get_locked_pool(session=session, tenant_id=tenant_id, pool_type=pool_type)
            if not pool:
                raise QuotaExceededError("Credit pool not found")
            if pool.quota_used < credits:
                raise RuntimeError("Reserved credits exceed recorded usage.")

            pool.quota_used -= credits
            session.commit()
            return credits

        try:
            cls._deduct_with_tenant_lock(tenant_id, release)
        except Exception:
            session.rollback()
            raise

    @classmethod
    def check_and_deduct_credits(
        cls,
        tenant_id: str,
        credits_required: int,
        pool_type: str | ProviderQuotaType = "trial",
        *,
        request_id: str | None = None,
        metadata: Mapping[str, object] | None = None,
        session: Session | None = None,
    ) -> int:
        """Deduct exactly the requested credits or raise without mutating the pool."""
        if credits_required <= 0:
            return 0
        normalized_pool_type = cls._normalize_pool_type(pool_type)

        if cls._use_billing_quota():
            from services.billing_service import BillingService

            resolved_request_id = request_id or str(uuid4())
            billing_metadata = cls._build_billing_metadata("credit_pool.check_and_deduct", metadata)
            result = BillingService.quota_reserve(
                tenant_id=tenant_id,
                feature_key=FEATURE_KEY_CREDIT_POOL,
                bucket=normalized_pool_type,
                request_id=resolved_request_id,
                amount=credits_required,
                meta=billing_metadata,
            )
            reservation_id = result.reservation_id
            if not reservation_id:
                raise QuotaExceededError("Insufficient credits remaining")
            try:
                BillingService.quota_commit(
                    tenant_id=tenant_id,
                    feature_key=FEATURE_KEY_CREDIT_POOL,
                    bucket=normalized_pool_type,
                    reservation_id=reservation_id,
                    actual_amount=credits_required,
                    meta=billing_metadata,
                )
            except Exception:
                try:
                    BillingService.quota_release(
                        tenant_id=tenant_id,
                        feature_key=FEATURE_KEY_CREDIT_POOL,
                        bucket=normalized_pool_type,
                        reservation_id=reservation_id,
                    )
                except Exception:
                    logger.warning(
                        "Failed to release reserved credit pool quota, tenant_id=%s, pool_type=%s, reservation_id=%s",
                        tenant_id,
                        normalized_pool_type,
                        reservation_id,
                        exc_info=True,
                    )
                raise
            return credits_required

        session = cls._require_session(session)

        def deduct() -> int:
            pool = cls._get_locked_pool(session=session, tenant_id=tenant_id, pool_type=normalized_pool_type)
            if not pool:
                raise QuotaExceededError("Credit pool not found")

            remaining_credits = pool.remaining_credits
            if remaining_credits <= 0:
                raise QuotaExceededError("No credits remaining")
            if remaining_credits < credits_required:
                raise QuotaExceededError("Insufficient credits remaining")

            pool.quota_used += credits_required
            session.commit()
            return credits_required

        try:
            return cls._deduct_with_tenant_lock(tenant_id, deduct)
        except QuotaExceededError:
            raise
        except Exception as caught_error:
            logger.exception("Failed to deduct credits for tenant %s", tenant_id)
            raise QuotaExceededError("Failed to deduct credits") from caught_error

    @classmethod
    def deduct_credits_capped(
        cls,
        tenant_id: str,
        credits_required: int,
        pool_type: str | ProviderQuotaType = "trial",
        *,
        request_id: str | None = None,
        metadata: Mapping[str, object] | None = None,
        session: Session | None = None,
    ) -> int:
        """Deduct up to the available balance and return the actual deducted credits."""
        if credits_required <= 0:
            return 0
        normalized_pool_type = cls._normalize_pool_type(pool_type)

        if cls._use_billing_quota():
            from services.billing_service import BillingService

            billing_metadata = cls._build_billing_metadata("credit_pool.deduct_capped", metadata)
            result = BillingService.quota_consume_capped(
                tenant_id=tenant_id,
                feature_key=FEATURE_KEY_CREDIT_POOL,
                bucket=normalized_pool_type,
                request_id=request_id or str(uuid4()),
                amount=credits_required,
                meta=billing_metadata,
            )
            return result.deducted

        session = cls._require_session(session)

        def deduct() -> int:
            pool = cls._get_locked_pool(session=session, tenant_id=tenant_id, pool_type=normalized_pool_type)
            if not pool:
                logger.warning("Credit pool not found, tenant_id=%s, pool_type=%s", tenant_id, normalized_pool_type)
                return 0

            deducted_credits = min(credits_required, pool.remaining_credits)
            if deducted_credits <= 0:
                return 0

            pool.quota_used += deducted_credits
            session.commit()
            return deducted_credits

        try:
            return cls._deduct_with_tenant_lock(tenant_id, deduct)
        except QuotaExceededError:
            raise
        except Exception as caught_error:
            logger.exception("Failed to deduct capped credits for tenant %s", tenant_id)
            raise QuotaExceededError("Failed to deduct credits") from caught_error