Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

3.2 KB
Raw Blame
from collections.abc import Sequence
from typing import Any, override

from core.moderation.base import Moderation, ModerationAction, ModerationInputsResult, ModerationOutputsResult


class KeywordsModeration(Moderation):
    name: str = "keywords"

    @classmethod
    @override
    def validate_config(cls, tenant_id: str, config: dict[str, Any]):
        """
        Validate the incoming form config data.

        :param tenant_id: the id of workspace
        :param config: the form config data
        :return:
        """
        cls._validate_inputs_and_outputs_config(config, True)

        if not config.get("keywords"):
            raise ValueError("keywords is required")

        if len(config.get("keywords", [])) > 10000:
            raise ValueError("keywords length must be less than 10000")

        keywords_rows = cls._keyword_rows(config["keywords"])
        if len(keywords_rows) > 100:
            raise ValueError("the number of rows for the keywords must be less than 100")

    @override
    def moderation_for_inputs(self, inputs: dict[str, Any], query: str = "") -> ModerationInputsResult:
        flagged = False
        preset_response = ""
        if self.config is None:
            raise ValueError("The config is not set.")

        if self.config["inputs_config"]["enabled"]:
            preset_response = self.config["inputs_config"]["preset_response"]

            if query:
                inputs["query__"] = query

            keywords_list = self._keyword_rows(self.config["keywords"])

            flagged = self._is_violated(inputs, keywords_list)

        return ModerationInputsResult(
            flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response
        )

    @override
    def moderation_for_outputs(self, text: str) -> ModerationOutputsResult:
        flagged = False
        preset_response = ""
        if self.config is None:
            raise ValueError("The config is not set.")

        if self.config["outputs_config"]["enabled"]:
            keywords_list = self._keyword_rows(self.config["keywords"])

            flagged = self._is_violated({"text": text}, keywords_list)
            preset_response = self.config["outputs_config"]["preset_response"]

        return ModerationOutputsResult(
            flagged=flagged, action=ModerationAction.DIRECT_OUTPUT, preset_response=preset_response
        )

    @staticmethod
    def _keyword_rows(keywords: str) -> list[str]:
        """Split the keywords config into stripped, non-empty rows.

        Rows keep surrounding whitespace from pasted configs (for example CRLF
        line endings) unless stripped: a keyword ending in "\r" never matches
        real text, and a whitespace-only row matches nearly every message.
        """
        return [row for row in (row.strip() for row in keywords.split("\n")) if row]

    def _is_violated(self, inputs: dict[str, Any], keywords_list: list[str]) -> bool:
        return any(self._check_keywords_in_value(keywords_list, value) for value in inputs.values())

    def _check_keywords_in_value(self, keywords_list: Sequence[str], value: Any) -> bool:
        return any(keyword.lower() in str(value).lower() for keyword in keywords_list)