RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
refactor: remove unused py code (#20388)
Jin Haib2d71edfc5bf
diff --git a/common/constants.py b/common/constants.pydeleted file mode 100644index 6080e687b9..0000000000--- a/common/constants.py+++ /dev/null@@ -1,393 +0,0 @@-#-# Copyright 2025 The InfiniFlow Authors. All Rights Reserved.-#-# Licensed under the Apache License, Version 2.0 (the "License");-# you may not use this file except in compliance with the License.-# You may obtain a copy of the License at-#-# http://www.apache.org/licenses/LICENSE-2.0-#-# Unless required by applicable law or agreed to in writing, software-# distributed under the License is distributed on an "AS IS" BASIS,-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.-# See the License for the specific language governing permissions and-# limitations under the License.-#--import os-from enum import Enum, IntEnum-from enum import StrEnum--SERVICE_CONF = "service_conf.yaml"-RAG_FLOW_SERVICE_NAME = "ragflow"-SANDBOX_ARTIFACT_BUCKET = os.environ.get("SANDBOX_ARTIFACT_BUCKET", "sandbox-artifacts")-SANDBOX_ARTIFACT_EXPIRE_DAYS = int(os.environ.get("SANDBOX_ARTIFACT_EXPIRE_DAYS", "7"))---class CustomEnum(Enum):- @classmethod- def valid(cls, value):- try:- cls(value)- return True- except BaseException:- return False-- @classmethod- def values(cls):- return [member.value for member in cls.__members__.values()]-- @classmethod- def names(cls):- return [member.name for member in cls.__members__.values()]---class RetCode(IntEnum, CustomEnum):- SUCCESS = 0- NOT_EFFECTIVE = 10- EXCEPTION_ERROR = 100- ARGUMENT_ERROR = 101- DATA_ERROR = 102- OPERATING_ERROR = 103- CONNECTION_ERROR = 105- RUNNING = 106- PERMISSION_ERROR = 108- AUTHENTICATION_ERROR = 109- BAD_REQUEST = 400- UNAUTHORIZED = 401- SERVER_ERROR = 500- FORBIDDEN = 403- NOT_FOUND = 404- CONFLICT = 409---class StatusEnum(Enum):- VALID = "1"- INVALID = "0"---class ActiveStatusEnum(Enum):- ACTIVE = "active"- INACTIVE = "inactive"- UNSUPPORTED = "unsupported"---class ModelVerifyStatusEnum(Enum):- SUCCESS = "success"- FAIL = "fail"- UNKNOWN = "unknown"---class ActiveEnum(Enum):- ACTIVE = "1"- INACTIVE = "0"---class LLMType(StrEnum):- CHAT = "chat"- EMBEDDING = "embedding"- ASR = "asr"- VISION = "vision"- RERANK = "rerank"- TTS = "tts"- OCR = "ocr"---class ModelTypeBinary(Enum):- CHAT = 0b0000001 # 1 << 0 = 1- EMBEDDING = 0b0000010 # 1 << 1 = 2- ASR = 0b0000100 # 1 << 2 = 4- VISION = 0b0001000 # 1 << 3 = 8- RERANK = 0b0010000 # 1 << 4 = 16- TTS = 0b0100000 # 1 << 5 = 32- OCR = 0b1000000 # 1 << 6 = 64---class TaskStatus(StrEnum):- UNSTART = "0"- RUNNING = "1"- CANCEL = "2"- DONE = "3"- FAIL = "4"- SCHEDULE = "5"---VALID_TASK_STATUS = {TaskStatus.UNSTART, TaskStatus.RUNNING, TaskStatus.CANCEL, TaskStatus.DONE, TaskStatus.FAIL, TaskStatus.SCHEDULE}---class ConnectorTaskType(StrEnum):- SYNC = "sync"- PRUNE = "prune"---class ParserType(StrEnum):- PRESENTATION = "presentation"- LAWS = "laws"- MANUAL = "manual"- PAPER = "paper"- RESUME = "resume"- BOOK = "book"- QA = "qa"- TABLE = "table"- NAIVE = "naive"- PICTURE = "picture"- ONE = "one"- AUDIO = "audio"- EMAIL = "email"- KG = "knowledge_graph"- TAG = "tag"---class FileSource(StrEnum):- LOCAL = ""- KNOWLEDGEBASE = "knowledgebase"- RSS = "rss"- SITEMAP = "sitemap"- S3 = "s3"- NOTION = "notion"- REST_API = "rest_api"- XQUIK = "xquik"- DISCORD = "discord"- CONFLUENCE = "confluence"- GMAIL = "gmail"- GOOGLE_DRIVE = "google_drive"- FEISHU_WIKI = "feishu_wiki"- JIRA = "jira"- SHAREPOINT = "sharepoint"- SLACK = "slack"- TEAMS = "teams"- WEBDAV = "webdav"- MOODLE = "moodle"- DROPBOX = "dropbox"- BOX = "box"- R2 = "r2"- OCI_STORAGE = "oci_storage"- GOOGLE_CLOUD_STORAGE = "google_cloud_storage"- AIRTABLE = "airtable"- ASANA = "asana"- GITHUB = "github"- GITLAB = "gitlab"- IMAP = "imap"- BITBUCKET = "bitbucket"- AZURE_DEVOPS = "azure_devops"- ZENDESK = "zendesk"- ZOTERO = "zotero"- SEAFILE = "seafile"- MYSQL = "mysql"- POSTGRESQL = "postgresql"- BIGQUERY = "bigquery"- DINGTALK_AI_TABLE = "dingtalk_ai_table"- ONEDRIVE = "onedrive"- OUTLOOK = "outlook"- SALESFORCE = "salesforce"- AZURE_BLOB = "azure_blob"---class PipelineTaskType(StrEnum):- PARSE = "Parse"- DOWNLOAD = "Download"- RAPTOR = "RAPTOR"- GRAPH_RAG = "GraphRAG"- MINDMAP = "Mindmap"- MEMORY = "Memory"- # Member name kept as ARTIFACT for back-compat; value is "Wiki" so the- # runtime task_type (``.lower()`` == "wiki") matches the wiki index/task type.- ARTIFACT = "Wiki"- SKILL = "Skill"- # KB-wide structure-graph merge tasks (rebuild_dataset_structure_graph_json).- STRUCTURE_GRAPH = "StructureGraph"- STRUCTURE_MINDMAP = "StructureMindmap"- TIMELINE = "Timeline"- SESSION_GRAPH = "SessionGraph"- SESSION_ESSENCE = "SessionEssence"- STRUCTURE = "Structure"---VALID_PIPELINE_TASK_TYPES = {- PipelineTaskType.PARSE,- PipelineTaskType.DOWNLOAD,- PipelineTaskType.RAPTOR,- PipelineTaskType.GRAPH_RAG,- PipelineTaskType.MINDMAP,- PipelineTaskType.ARTIFACT,- PipelineTaskType.SKILL,- PipelineTaskType.STRUCTURE_GRAPH,- PipelineTaskType.STRUCTURE_MINDMAP,- PipelineTaskType.TIMELINE,- PipelineTaskType.SESSION_GRAPH,- PipelineTaskType.SESSION_ESSENCE,- PipelineTaskType.STRUCTURE,-}---class MCPServerType(StrEnum):- SSE = "sse"- STREAMABLE_HTTP = "streamable-http"---VALID_MCP_SERVER_TYPES = {MCPServerType.SSE, MCPServerType.STREAMABLE_HTTP}---class Storage(Enum):- MINIO = 1- AZURE_SPN = 2- AZURE_SAS = 3- AWS_S3 = 4- OSS = 5- OPENDAL = 6- GCS = 7---class MemoryType(Enum):- RAW = 0b0001 # 1 << 0 = 1 (0b00000001)- SEMANTIC = 0b0010 # 1 << 1 = 2 (0b00000010)- EPISODIC = 0b0100 # 1 << 2 = 4 (0b00000100)- PROCEDURAL = 0b1000 # 1 << 3 = 8 (0b00001000)---class MemoryStorageType(StrEnum):- TABLE = "table"- GRAPH = "graph"---class ForgettingPolicy(StrEnum):- FIFO = "FIFO"---# environment-# ENV_STRONG_TEST_COUNT = "STRONG_TEST_COUNT"-# ENV_RAGFLOW_SECRET_KEY = "RAGFLOW_SECRET_KEY"-# ENV_REGISTER_ENABLED = "REGISTER_ENABLED"-# ENV_DOC_ENGINE = "DOC_ENGINE"-# ENV_SANDBOX_ENABLED = "SANDBOX_ENABLED"-# ENV_SANDBOX_HOST = "SANDBOX_HOST"-# ENV_MAX_CONTENT_LENGTH = "MAX_CONTENT_LENGTH"-# ENV_COMPONENT_EXEC_TIMEOUT = "COMPONENT_EXEC_TIMEOUT"-# ENV_TRINO_USE_TLS = "TRINO_USE_TLS"-# ENV_MAX_FILE_NUM_PER_USER = "MAX_FILE_NUM_PER_USER"-# ENV_MACOS = "MACOS"-# ENV_RAGFLOW_DEBUGPY_LISTEN = "RAGFLOW_DEBUGPY_LISTEN"-# ENV_WERKZEUG_RUN_MAIN = "WERKZEUG_RUN_MAIN"-# ENV_DISABLE_SDK = "DISABLE_SDK"-# ENV_ENABLE_TIMEOUT_ASSERTION = "ENABLE_TIMEOUT_ASSERTION"-# ENV_LOG_LEVELS = "LOG_LEVELS"-# ENV_TENSORRT_DLA_SVR = "TENSORRT_DLA_SVR"-# ENV_OCR_GPU_MEM_LIMIT_MB = "OCR_GPU_MEM_LIMIT_MB"-# ENV_OCR_ARENA_EXTEND_STRATEGY = "OCR_ARENA_EXTEND_STRATEGY"-# ENV_MAX_CONCURRENT_PROCESS_AND_EXTRACT_CHUNK = "MAX_CONCURRENT_PROCESS_AND_EXTRACT_CHUNK"-# ENV_MAX_MAX_CONCURRENT_CHATS = "MAX_CONCURRENT_CHATS"-# ENV_RAGFLOW_MCP_BASE_URL = "RAGFLOW_MCP_BASE_URL"-# ENV_RAGFLOW_MCP_HOST = "RAGFLOW_MCP_HOST"-# ENV_RAGFLOW_MCP_PORT = "RAGFLOW_MCP_PORT"-# ENV_RAGFLOW_MCP_LAUNCH_MODE = "RAGFLOW_MCP_LAUNCH_MODE"-# ENV_RAGFLOW_MCP_HOST_API_KEY = "RAGFLOW_MCP_HOST_API_KEY"-# ENV_MINERU_EXECUTABLE = "MINERU_EXECUTABLE"-# ENV_MINERU_APISERVER = "MINERU_APISERVER"-# ENV_MINERU_OUTPUT_DIR = "MINERU_OUTPUT_DIR"-# ENV_MINERU_BACKEND = "MINERU_BACKEND"-# ENV_MINERU_DELETE_OUTPUT = "MINERU_DELETE_OUTPUT"-# ENV_SOMARK_BASE_URL = "SOMARK_BASE_URL"-# ENV_SOMARK_API_KEY = "SOMARK_API_KEY"-# ENV_SOMARK_IMAGE_FORMAT = "SOMARK_IMAGE_FORMAT"-# ENV_SOMARK_FORMULA_FORMAT = "SOMARK_FORMULA_FORMAT"-# ENV_SOMARK_TABLE_FORMAT = "SOMARK_TABLE_FORMAT"-# ENV_SOMARK_CS_FORMAT = "SOMARK_CS_FORMAT"-# ENV_SOMARK_ENABLE_TEXT_CROSS_PAGE = "SOMARK_ENABLE_TEXT_CROSS_PAGE"-# ENV_SOMARK_ENABLE_TABLE_CROSS_PAGE = "SOMARK_ENABLE_TABLE_CROSS_PAGE"-# ENV_SOMARK_ENABLE_TITLE_LEVEL_RECOGNITION = "SOMARK_ENABLE_TITLE_LEVEL_RECOGNITION"-# ENV_SOMARK_ENABLE_INLINE_IMAGE = "SOMARK_ENABLE_INLINE_IMAGE"-# ENV_SOMARK_ENABLE_TABLE_IMAGE = "SOMARK_ENABLE_TABLE_IMAGE"-# ENV_SOMARK_ENABLE_IMAGE_UNDERSTANDING = "SOMARK_ENABLE_IMAGE_UNDERSTANDING"-# ENV_SOMARK_KEEP_HEADER_FOOTER = "SOMARK_KEEP_HEADER_FOOTER"-# ENV_DOCLING_SERVER_URL = "DOCLING_SERVER_URL"-# ENV_DOCLING_OUTPUT_DIR = "DOCLING_OUTPUT_DIR"-# ENV_DOCLING_DELETE_OUTPUT = "DOCLING_DELETE_OUTPUT"-# ENV_TCADP_OUTPUT_DIR = "TCADP_OUTPUT_DIR"-# ENV_LM_TIMEOUT_SECONDS = "LM_TIMEOUT_SECONDS"-# ENV_LLM_MAX_RETRIES = "LLM_MAX_RETRIES"-# ENV_LLM_BASE_DELAY = "LLM_BASE_DELAY"-# ENV_OLLAMA_KEEP_ALIVE = "OLLAMA_KEEP_ALIVE"-# ENV_DOC_BULK_SIZE = "DOC_BULK_SIZE"-# ENV_EMBEDDING_BATCH_SIZE = "EMBEDDING_BATCH_SIZE"-# ENV_MAX_CONCURRENT_TASKS = "MAX_CONCURRENT_TASKS"-# ENV_MAX_CONCURRENT_CHUNK_BUILDERS = "MAX_CONCURRENT_CHUNK_BUILDERS"-# ENV_MAX_CONCURRENT_MINIO = "MAX_CONCURRENT_MINIO"-# ENV_WORKER_HEARTBEAT_TIMEOUT = "WORKER_HEARTBEAT_TIMEOUT"-# ENV_TRACE_MALLOC_ENABLED = "TRACE_MALLOC_ENABLED"--PAGERANK_FLD = "pagerank_fea"-SVR_QUEUE_NAME = "te"-SVR_CONSUMER_GROUP_NAME = "rag_flow_svr_task_broker"-TAG_FLD = "tag_feas"--# Maximum page number used as "unlimited" sentinel value.-# Parsing layer (chunk/Pdf.__call__) uses MAXIMUM_PAGE_NUMBER.-# Task/DB layer (Task model) uses MAXIMUM_PAGE_NUMBER * 1000 to avoid collision with user-specified page ranges.-MAXIMUM_PAGE_NUMBER = 100000-MAXIMUM_TASK_PAGE_NUMBER = MAXIMUM_PAGE_NUMBER * 1000---MINERU_ENV_KEYS = ["MINERU_APISERVER", "MINERU_OUTPUT_DIR", "MINERU_BACKEND", "MINERU_SERVER_URL", "MINERU_DELETE_OUTPUT"]-MINERU_DEFAULT_CONFIG = {- "MINERU_APISERVER": "",- "MINERU_OUTPUT_DIR": "",- "MINERU_BACKEND": "pipeline",- "MINERU_SERVER_URL": "",- "MINERU_DELETE_OUTPUT": 1,-}--PADDLEOCR_ENV_KEYS = ["PADDLEOCR_BASE_URL", "PADDLEOCR_API_URL", "PADDLEOCR_ACCESS_TOKEN", "PADDLEOCR_ALGORITHM"]-PADDLEOCR_DEFAULT_CONFIG = {- "PADDLEOCR_BASE_URL": "",- "PADDLEOCR_API_URL": "",- "PADDLEOCR_ACCESS_TOKEN": None,- "PADDLEOCR_ALGORITHM": "PaddleOCR-VL",-}--OPENDATALOADER_ENV_KEYS = ["OPENDATALOADER_APISERVER"]-OPENDATALOADER_DEFAULT_CONFIG = {- "OPENDATALOADER_APISERVER": "",-}--SOMARK_ENV_KEYS = [- "SOMARK_BASE_URL",- "SOMARK_API_KEY",- "SOMARK_IMAGE_FORMAT",- "SOMARK_FORMULA_FORMAT",- "SOMARK_TABLE_FORMAT",- "SOMARK_CS_FORMAT",- "SOMARK_ENABLE_TEXT_CROSS_PAGE",- "SOMARK_ENABLE_TABLE_CROSS_PAGE",- "SOMARK_ENABLE_TITLE_LEVEL_RECOGNITION",- "SOMARK_ENABLE_INLINE_IMAGE",- "SOMARK_ENABLE_TABLE_IMAGE",- "SOMARK_ENABLE_IMAGE_UNDERSTANDING",- "SOMARK_KEEP_HEADER_FOOTER",-]-SOMARK_DEFAULT_CONFIG = {- "SOMARK_BASE_URL": "https://somark.cn/api/v1",- "SOMARK_API_KEY": "",- "SOMARK_IMAGE_FORMAT": "url",- "SOMARK_FORMULA_FORMAT": "latex",- "SOMARK_TABLE_FORMAT": "html",- "SOMARK_CS_FORMAT": "image",- "SOMARK_ENABLE_TEXT_CROSS_PAGE": 0,- "SOMARK_ENABLE_TABLE_CROSS_PAGE": 0,- "SOMARK_ENABLE_TITLE_LEVEL_RECOGNITION": 0,- "SOMARK_ENABLE_INLINE_IMAGE": 1,- "SOMARK_ENABLE_TABLE_IMAGE": 1,- "SOMARK_ENABLE_IMAGE_UNDERSTANDING": 1,- "SOMARK_KEEP_HEADER_FOOTER": 0,-}-MISTRAL_OCR_ENV_KEYS = [- "MISTRAL_OCR_BASE_URL",- "MISTRAL_OCR_API_KEY",- "MISTRAL_OCR_TABLE_FORMAT",- "MISTRAL_OCR_KEEP_HEADER_FOOTER",-]-MISTRAL_OCR_DEFAULT_CONFIG = {- "MISTRAL_OCR_BASE_URL": "https://api.mistral.ai/v1",- "MISTRAL_OCR_API_KEY": "",- "MISTRAL_OCR_TABLE_FORMAT": "html",- "MISTRAL_OCR_KEEP_HEADER_FOOTER": 0,-}