Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

1.6 KB
Raw Blame
from enum import StrEnum
from typing import Literal

from pydantic import AliasChoices, BaseModel, Field


class ParentMode(StrEnum):
    FULL_DOC = "full-doc"
    PARAGRAPH = "paragraph"


type PreProcessingRuleKey = Literal["remove_stopwords", "remove_extra_spaces", "remove_urls_emails"]


class PreProcessingRule(BaseModel):
    id: PreProcessingRuleKey = Field(description="Rule identifier.")
    enabled: bool = Field(description="Whether this preprocessing rule is enabled.")


class Segmentation(BaseModel):
    # TODO: there are internally mismatched / inconsistent naming
    # between `separator` and `delimiter` across the codebase.
    # Taking `separator` as the canonical.
    separator: str = Field(
        default="\n",
        description="Custom separator for splitting text.",
        validation_alias=AliasChoices("separator", "delimiter"),
    )
    max_tokens: int = Field(description="Maximum token count per chunk.")
    chunk_overlap: int = Field(default=0, ge=0, description="Token overlap between chunks.")


class Rule(BaseModel):
    pre_processing_rules: list[PreProcessingRule] | None = Field(
        default=None,
        description="Pre-processing rules to apply before segmentation.",
    )
    segmentation: Segmentation | None = Field(default=None, description="Parent chunk segmentation settings.")
    parent_mode: Literal["full-doc", "paragraph"] | None = Field(
        default=None,
        description="Parent-child segmentation mode.",
    )
    subchunk_segmentation: Segmentation | None = Field(
        default=None,
        description="Child chunk segmentation settings.",
    )