Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

2.9 KB
Raw Blame
import logging
from collections.abc import Mapping
from typing import Any

from dify_trace_sdk.emitter import TraceScope
from dify_trace_sdk.events import ModerationChecked, json_safe

from core.app.app_config.entities import AppConfig
from core.app.entities.app_invoke_entities import get_credit_usage_app_type
from core.model_context import use_credit_usage_metadata
from core.moderation.base import ModerationAction, ModerationError
from core.moderation.factory import ModerationFactory
from libs.datetime_utils import measure_time

logger = logging.getLogger(__name__)


class InputModeration:
    def check(
        self,
        app_id: str,
        tenant_id: str,
        app_config: AppConfig,
        inputs: Mapping[str, Any],
        query: str,
        message_id: str,
        trace_scope: TraceScope | None,
    ) -> tuple[bool, Mapping[str, Any], str]:
        """
        Process sensitive_word_avoidance.
        :param app_id: app id
        :param tenant_id: tenant id
        :param app_config: app config
        :param inputs: inputs
        :param query: query
        :param message_id: message id
        :param trace_scope: where the moderation verdict is reported, if anywhere
        :return:
        """
        inputs = dict(inputs)
        if not app_config.sensitive_word_avoidance:
            return False, inputs, query

        sensitive_word_avoidance_config = app_config.sensitive_word_avoidance
        moderation_type = sensitive_word_avoidance_config.type

        with use_credit_usage_metadata({"app_type": get_credit_usage_app_type(app_config.app_mode)}):
            moderation_factory = ModerationFactory(
                name=moderation_type, app_id=app_id, tenant_id=tenant_id, config=sensitive_word_avoidance_config.config
            )

            with measure_time() as timer:
                moderation_result = moderation_factory.moderation_for_inputs(inputs, query)

        if trace_scope is not None:
            trace_scope.emit(
                ModerationChecked(
                    message_id=message_id,
                    inputs={key: json_safe(value) for key, value in inputs.items()},
                    query=moderation_result.query,
                    flagged=moderation_result.flagged,
                    action=moderation_result.action.value,
                    preset_response=moderation_result.preset_response,
                    started_at=timer.started_at,
                    ended_at=timer.ended_at,
                )
            )

        if not moderation_result.flagged:
            return False, inputs, query

        if moderation_result.action == ModerationAction.DIRECT_OUTPUT:
            raise ModerationError(moderation_result.preset_response)
        elif moderation_result.action == ModerationAction.OVERRIDDEN:
            inputs = moderation_result.inputs
            query = moderation_result.query

        return True, inputs, query