RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
Remove internal cli 'insert chunks from file' and 'insert metadata from file' (#19591)
qinling0210aa78e8d224e0
diff --git a/admin/client/parser.py b/admin/client/parser.pyindex 1cdb206b26..c4ad70410c 100644--- a/admin/client/parser.py+++ b/admin/client/parser.py@@ -97,8 +97,6 @@ sql_command: login_user| search_on_datasets| get_chunk| list_chunks- | insert_dataset_from_file- | insert_metadata_from_file| update_chunk| set_metadata| remove_tags@@ -223,11 +221,9 @@ TABLE: "TABLE"iCHUNK: "CHUNK"iCHUNKS: "CHUNKS"iGET: "GET"i-INSERT: "INSERT"iPAGE: "PAGE"iKEYWORDS: "KEYWORDS"iAVAILABLE: "AVAILABLE"i-FILE: "FILE"iUPDATE: "UPDATE"iREMOVE: "REMOVE"iTAGS: "TAGS"i@@ -377,8 +373,6 @@ create_dataset_table: CREATE DATASET TABLE quoted_string VECTOR SIZE NUMBER ";"drop_dataset_table: DROP DATASET TABLE quoted_string ";"create_metadata_table: CREATE METADATA TABLE ";"drop_metadata_table: DROP METADATA TABLE ";"-insert_dataset_from_file: INSERT DATASET FROM FILE quoted_string ";"-insert_metadata_from_file: INSERT METADATA FROM FILE quoted_string ";"generate_nav_for_dataset: GENERATE NAVIGATION OF DATASET quoted_string ";"navigation_search: NAVIGATION SEARCH quoted_string IN DATASET quoted_string MODE quoted_string (TOPK NUMBER)? ";"update_chunk: UPDATE CHUNK quoted_string OF DATASET quoted_string SET quoted_string ";"@@ -788,14 +782,6 @@ class RAGFlowCLITransformer(Transformer):chunk_id = items[2].children[0].strip("'\"")return {"type": "get_chunk", "chunk_id": chunk_id}- def insert_dataset_from_file(self, items):- file_path = items[4].children[0].strip("'\"")- return {"type": "insert_dataset_from_file", "file_path": file_path}-- def insert_metadata_from_file(self, items):- file_path = items[4].children[0].strip("'\"")- return {"type": "insert_metadata_from_file", "file_path": file_path}-def generate_nav_for_dataset(self, items):dataset_id = items[4].children[0].strip("'\"")return {"type": "generate_nav_for_dataset", "dataset_id": dataset_id}diff --git a/admin/client/ragflow_client.py b/admin/client/ragflow_client.pyindex ead4eec1ab..bf9f025310 100644--- a/admin/client/ragflow_client.py+++ b/admin/client/ragflow_client.py@@ -1645,46 +1645,6 @@ class RAGFlowClient:else:print(f"Fail to get chunk, code: {response.status_code}, body: {response.text}")- # Internal- def insert_dataset_from_file(self, command_dict):- if self.server_type != "user":- print("This command is only allowed in USER mode")- return-- file_path = command_dict["file_path"]- payload = {"file_path": file_path}- response = self.http_client.request("POST", "/kb/insert_from_file", json_body=payload, use_api_base=False, auth_kind="web")- if response.status_code == 200:- res_json = response.json()- if res_json["code"] == 0:- print(f"Success to insert dataset from file: {file_path}")- if res_json.get("data"):- self._print_key_value(res_json["data"])- else:- print(f"Fail to insert dataset from file, code: {res_json['code']}, message: {res_json['message']}")- else:- print(f"Fail to insert dataset from file, code: {response.status_code}, body: {response.text}")-- # Internal- def insert_metadata_from_file(self, command_dict):- if self.server_type != "user":- print("This command is only allowed in USER mode")- return-- file_path = command_dict["file_path"]- payload = {"file_path": file_path}- response = self.http_client.request("POST", "/tenant/insert_metadata_from_file", json_body=payload, use_api_base=False, auth_kind="web")- if response.status_code == 200:- res_json = response.json()- if res_json["code"] == 0:- print(f"Success to insert metadata from file: {file_path}")- if res_json.get("data"):- self._print_key_value(res_json["data"])- else:- print(f"Fail to insert metadata from file, code: {res_json['code']}, message: {res_json['message']}")- else:- print(f"Fail to insert metadata from file, code: {response.status_code}, body: {response.text}")-def update_chunk(self, command_dict):if self.server_type != "user":print("This command is only allowed in USER mode")@@ -2290,10 +2250,6 @@ def run_command(client: RAGFlowClient, command_dict: dict):return client.search_on_datasets(command_dict)case "get_chunk":return client.get_chunk(command_dict)- case "insert_dataset_from_file":- return client.insert_dataset_from_file(command_dict)- case "insert_metadata_from_file":- return client.insert_metadata_from_file(command_dict)case "update_chunk":return client.update_chunk(command_dict)case "set_metadata":diff --git a/internal/cli/cli_http.go b/internal/cli/cli_http.goindex d1d77165ff..bd273788a0 100644--- a/internal/cli/cli_http.go+++ b/internal/cli/cli_http.go@@ -528,10 +528,6 @@ func (c *CLI) ExecuteUserCommand(cmd *Command) (ResponseIf, error) {return c.DevCreateMetadataStoreCommand(cmd)case "dev_drop_metadata_store":return c.DevDropMetadataStoreCommand(cmd)- case "dev_insert_chunks_from_file":- return c.DevInsertChunksFromFileCommand(cmd)- case "dev_insert_metadata_from_file":- return c.DevInsertMetadataFromFileCommand(cmd)case "dev_update_chunk":return c.DevUpdateChunkCommand(cmd)case "dev_get_chunk":diff --git a/internal/cli/dev_parser.go b/internal/cli/dev_parser.goindex 757707aeff..06f91e509f 100644--- a/internal/cli/dev_parser.go+++ b/internal/cli/dev_parser.go@@ -21,89 +21,6 @@ import ("strconv")-// INSERT CHUNKS FROM FILE "file_path"-// INSERT METADATA FROM FILE "file_path"-func (p *Parser) parseDevInsertCommand() (*Command, error) {- p.nextToken() // consume INSERT-- // Expect CHUNKS or METADATA- if p.curToken.Type == TokenChunks {- return p.parseInsertChunksFromFile()- }- if p.curToken.Type == TokenMetadata {- return p.parseInsertMetadataFromFile()- }- return nil, fmt.Errorf("expected CHUNKS or METADATA after INSERT, got %s", p.curToken.Value)-}--// Internal CLI for GO-// parseInsertChunksFromFile parses: INSERT CHUNKS FROM FILE "file_path"-func (p *Parser) parseInsertChunksFromFile() (*Command, error) {- p.nextToken() // consume CHUNKS-- // Expect FROM- if p.curToken.Type != TokenFrom {- return nil, fmt.Errorf("expected FROM, got %s", p.curToken.Value)- }- p.nextToken()-- // Expect FILE- if p.curToken.Type != TokenFile {- return nil, fmt.Errorf("expected FILE, got %s", p.curToken.Value)- }- p.nextToken()-- // Get file path (quoted string)- filePath, err := p.parseQuotedString()- if err != nil {- return nil, err- }-- cmd := NewCommand("dev_insert_chunks_from_file")- cmd.Params["file_path"] = filePath-- p.nextToken()- // Semicolon is optional- if p.curToken.Type == TokenSemicolon {- p.nextToken()- }- return cmd, nil-}--// Internal CLI for GO-// parseInsertMetadataFromFile parses: INSERT METADATA FROM FILE "file_path"-func (p *Parser) parseInsertMetadataFromFile() (*Command, error) {- p.nextToken() // consume METADATA-- // Expect FROM- if p.curToken.Type != TokenFrom {- return nil, fmt.Errorf("expected FROM, got %s", p.curToken.Value)- }- p.nextToken()-- // Expect FILE- if p.curToken.Type != TokenFile {- return nil, fmt.Errorf("expected FILE, got %s", p.curToken.Value)- }- p.nextToken()-- // Get file path (quoted string)- filePath, err := p.parseQuotedString()- if err != nil {- return nil, err- }-- cmd := NewCommand("dev_insert_metadata_from_file")- cmd.Params["file_path"] = filePath-- p.nextToken()- // Semicolon is optional- if p.curToken.Type == TokenSemicolon {- p.nextToken()- }- return cmd, nil-}-// parseDevGetCommand parses: GET CHUNK or GET METADATAfunc (p *Parser) parseDevGetCommand() (*Command, error) {p.nextToken() // consume GETdiff --git a/internal/cli/lexer.go b/internal/cli/lexer.goindex 93fc0a5324..07c618be95 100644--- a/internal/cli/lexer.go+++ b/internal/cli/lexer.go@@ -447,8 +447,6 @@ func (l *Lexer) lookupIdent(ident string) Token {return Token{Type: TokenDisable, Value: ident}case "ENABLE":return Token{Type: TokenEnable, Value: ident}- case "INSERT":- return Token{Type: TokenInsert, Value: ident}case "FILE":return Token{Type: TokenFile, Value: ident}case "USE":diff --git a/internal/cli/parser.go b/internal/cli/parser.goindex 6549b733a1..426634c637 100644--- a/internal/cli/parser.go+++ b/internal/cli/parser.go@@ -176,9 +176,6 @@ func (p *Parser) parseUserCommand() (*Command, error) {return p.parseAPIResetCommands()case TokenImport:return p.parseAPIImport()- case TokenInsert:- // Use for development only- return p.parseDevInsertCommand()case TokenRetrieve:return p.parseAPIRetrieve()case TokenParse:diff --git a/internal/cli/types.go b/internal/cli/types.goindex f6f48c435c..2ebdc71876 100644--- a/internal/cli/types.go+++ b/internal/cli/types.go@@ -159,7 +159,6 @@ const (TokenMaxTokenLSTokenCat- TokenInsertTokenFileTokenMetadataTokenTablediff --git a/internal/cli/user_command.go b/internal/cli/user_command.goindex caea23aa45..46ed20cdc8 100644--- a/internal/cli/user_command.go+++ b/internal/cli/user_command.go@@ -2895,95 +2895,6 @@ func (c *CLI) APIAddCustomModelCommand(cmd *Command) (ResponseIf, error) {return HandleSimpleResponse(resp, "add custom model")}-// DevInsertChunksFromFileCommand inserts chunks from a JSON file-func (c *CLI) DevInsertChunksFromFileCommand(cmd *Command) (ResponseIf, error) {- if c.Config.CLIMode != APIMode {- return nil, fmt.Errorf("this command is only allowed in USER mode")- }- filePath, ok := cmd.Params["file_path"].(string)- if !ok {- return nil, fmt.Errorf("file_path not provided")- }-- payload := map[string]interface{}{- "file_path": filePath,- }-- resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/tenant/dev_insert_chunks_from_file", "web", nil, payload)- if err != nil {- return nil, fmt.Errorf("failed to insert dataset from file: %w", err)- }-- if resp.StatusCode != 200 {- return nil, fmt.Errorf("failed to insert dataset from file: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))- }-- resJSON, err := resp.JSON()- if err != nil {- return nil, fmt.Errorf("invalid JSON response: %w", err)- }-- code, ok := resJSON["code"].(float64)- if !ok {- return nil, fmt.Errorf("invalid response format: code is not a number")- }-- var result SimpleResponse- result.Code = int(code)- if result.Code == 0 {- result.Message = fmt.Sprintf("Success to insert dataset from file: %s", filePath)- } else {- result.Message = fmt.Sprintf("Failed to insert dataset from file: %v", resJSON)- }- result.Duration = 0- return &result, nil-}--// DevInsertMetadataFromFileCommand inserts metadata from a JSON file-func (c *CLI) DevInsertMetadataFromFileCommand(cmd *Command) (ResponseIf, error) {- if c.Config.CLIMode != APIMode {- return nil, fmt.Errorf("this command is only allowed in USER mode")- }-- filePath, ok := cmd.Params["file_path"].(string)- if !ok {- return nil, fmt.Errorf("file_path not provided")- }-- payload := map[string]interface{}{- "file_path": filePath,- }-- resp, err := c.APIServerClientMap[c.Config.APIClientConfig.CurrentAPIServer].Request("POST", "/tenant/dev_insert_metadata_from_file", "web", nil, payload)- if err != nil {- return nil, fmt.Errorf("failed to insert metadata from file: %w", err)- }-- if resp.StatusCode != 200 {- return nil, fmt.Errorf("failed to insert metadata from file: HTTP %d, body: %s", resp.StatusCode, string(resp.Body))- }-- resJSON, err := resp.JSON()- if err != nil {- return nil, fmt.Errorf("invalid JSON response: %w", err)- }-- code, ok := resJSON["code"].(float64)- if !ok {- return nil, fmt.Errorf("invalid response format: code is not a number")- }-- var result SimpleResponse- result.Code = int(code)- if result.Code == 0 {- result.Message = fmt.Sprintf("Success to insert metadata from file: %s", filePath)- } else {- result.Message = fmt.Sprintf("Failed to insert metadata from file: %v", resJSON)- }- result.Duration = 0- return &result, nil-}-// DevUpdateChunkCommand updates a chunk in a datasetfunc (c *CLI) DevUpdateChunkCommand(cmd *Command) (ResponseIf, error) {if c.Config.CLIMode != APIMode {diff --git a/internal/cli/user_parser.go b/internal/cli/user_parser.goindex 8eb149b881..b33587ac2b 100644--- a/internal/cli/user_parser.go+++ b/internal/cli/user_parser.go@@ -2685,8 +2685,6 @@ func (p *Parser) parseUserStatement() (*Command, error) {return p.parseAPIListCommands()case TokenImport:return p.parseAPIImport()- case TokenInsert:- return p.parseDevInsertCommand()case TokenRetrieve:return p.parseAPIRetrieve()default:diff --git a/internal/development.md b/internal/development.mdindex 12c59c8ca9..4fcce80c0a 100644--- a/internal/development.md+++ b/internal/development.md@@ -659,11 +659,6 @@ RAGFlow(api/default)> ocr with 'paddleocr-vl-0.9b@test@baidu' file './internal/tRAGFlow(api/default)> CREATE CHUNK STORE FOR DATASET 'test' VECTOR SIZE 384```-- Insert data from JSON files-```-RAGFlow(api/default)> INSERT CHUNKS FROM FILE 'insert_kb.json'-```-- Update a chunk's content```RAGFlow(api/default)> UPDATE CHUNK 'deb165dc6a732a64' OF DOCUMENT 'bbe55942535e11f1bc5184ba59049aa3' IN DATASET 'test' SET '{"content": "Updated chunk content here", "important_keywords": ["keyword1", "keyword2"], "questions": ["What is this about?", "Why is it important?"], "available": true, "tag_kwd": ["tag5", "tag2"]}'@@ -706,10 +701,6 @@ RAGFlow(api/default)> GET CHUNK '29cc4f6d7a5c6e7c' OF DATASET 'test' DOCUMENT 'bRAGFlow(api/default)> CREATE METADATA STORE```-- Insert metadata from JSON files-```-RAGFlow(api/default)> INSERT METADATA FROM FILE 'insert_metadata.json'-```- Set metadata for a document```RAGFlow(api/default)> SET METADATA OF DOCUMENT 'bbe55942535e11f1bc5184ba59049aa3' TO '{"author": ["John", "Tom"], "category": "tech"}';diff --git a/internal/handler/tenant.go b/internal/handler/tenant.goindex 14bccb7190..8792fc34eb 100644--- a/internal/handler/tenant.go+++ b/internal/handler/tenant.go@@ -17,14 +17,11 @@package handlerimport (- "encoding/json""net/http"- "os""github.com/gin-gonic/gin""ragflow/internal/common"- "ragflow/internal/engine""ragflow/internal/service"dataset "ragflow/internal/service/dataset")@@ -322,160 +319,6 @@ func (h *TenantHandler) DeleteChunkStore(c *gin.Context) {common.SuccessNoData(c, "success")}-// InsertChunksFromFileRequest request for inserting chunks from file-type InsertChunksFromFileRequest struct {- FilePath string `json:"file_path" binding:"required"`-}--// InsertChunksFromFile @Summary Insert chunks into dataset from JSON file-// @Description Internal: Insert chunks into dataset table from a JSON file-// @Tags tenants-// @Security ApiKeyAuth-// @Param request body InsertChunksFromFileRequest true "insert chunks request"-// @Success 200 {object} map[string]interface{}-// @Router /v1/tenant/dev_insert_chunks_from_file [post]-func (h *TenantHandler) InsertChunksFromFile(c *gin.Context) {- _, errorCode, errorMessage := GetUser(c)- if errorCode != common.CodeSuccess {- common.ErrorWithCode(c, errorCode, errorMessage)- return- }-- var req InsertChunksFromFileRequest- if err := c.ShouldBindJSON(&req); err != nil {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, err.Error())- return- }-- if req.FilePath == "" {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "file_path is required")- return- }-- // Read the JSON file- // codeql[go/path-injection] False positive: req.FilePath is the- // JSON file path the operator configured (tenant import flow). The- // OS access check enforces permissions, and the handler is gated- // to admin/owner roles upstream.- data, err := os.ReadFile(req.FilePath)- if err != nil {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "failed to read file: "+err.Error())- return- }-- // Parse JSON - format: {"index_name"/"table_name": ..., "knowledgebase_id": ..., "chunks": [...]}- var debugFormat struct {- IndexName string `json:"index_name"`- TableName string `json:"table_name"`- KnowledgebaseID string `json:"knowledgebase_id"`- Chunks []map[string]interface{} `json:"chunks"`- }-- if err = json.Unmarshal(data, &debugFormat); err != nil || debugFormat.Chunks == nil {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "invalid JSON format: expected {\"index_name\"/\"table_name\": ..., \"knowledgebase_id\": ..., \"chunks\": [...]}")- return- }-- if len(debugFormat.Chunks) == 0 {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "no chunks found in file")- return- }-- // Support both index_name (ES) and table_name (Infinity) in JSON- indexName := debugFormat.IndexName- if indexName == "" {- indexName = debugFormat.TableName- }-- // Get the document engine and insert- docEngine := engine.Get()- result, err := docEngine.InsertChunks(c.Request.Context(), debugFormat.Chunks, indexName, debugFormat.KnowledgebaseID)- if err != nil {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "failed to insert into dataset: "+err.Error())- return- }-- common.SuccessWithData(c, result, "success")-}--// InsertMetadataFromFileRequest request for inserting metadata from file-type InsertMetadataFromFileRequest struct {- FilePath string `json:"file_path" binding:"required"`-}--// InsertMetadataFromFile @Summary Insert document metadata from JSON file-// @Description Internal: Insert metadata into tenant's metadata table from a JSON file-// @Tags tenants-// @Security ApiKeyAuth-// @Param request body InsertMetadataFromFileRequest true "insert metadata request"-// @Success 200 {object} map[string]interface{}-// @Router /v1/tenant/dev_insert_metadata_from_file [post]-func (h *TenantHandler) InsertMetadataFromFile(c *gin.Context) {- user, errorCode, errorMessage := GetUser(c)- if errorCode != common.CodeSuccess {- common.ErrorWithCode(c, errorCode, errorMessage)- return- }-- var req InsertMetadataFromFileRequest- if err := c.ShouldBindJSON(&req); err != nil {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, err.Error())- return- }-- if req.FilePath == "" {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "file_path is required")- return- }-- // Read the JSON file- // codeql[go/path-injection] False positive: req.FilePath is the- // path the operator configured (tenant import flow). The- // OS access check enforces permissions, and the handler is gated- // to admin/owner roles upstream.- data, err := os.ReadFile(req.FilePath)- if err != nil {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "failed to read file: "+err.Error())- return- }-- // Parse JSON - format: {"chunks": [...]}- var inputFormat struct {- Chunks []map[string]interface{} `json:"chunks"`- }-- if err = json.Unmarshal(data, &inputFormat); err != nil || inputFormat.Chunks == nil {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "invalid JSON format: expected {\"chunks\": [...]}")- return- }-- if len(inputFormat.Chunks) == 0 {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "no chunks found in file")- return- }-- // Use user.ID as tenant ID (user IS the tenant in user mode)- tenantID := user.ID-- // Ensure the metadata store exists before writing (service-layer- // create-on-first-write logic; the engine layer assumes it exists).- metadataSvc := service.NewMetadataService()- if err := metadataSvc.EnsureMetadataStore(c.Request.Context(), tenantID); err != nil {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "failed to ensure metadata store: "+err.Error())- return- }-- // Get the document engine and insert- docEngine := engine.Get()- result, err := docEngine.InsertMetadata(c.Request.Context(), inputFormat.Chunks, tenantID)- if err != nil {- common.ResponseWithHttpCodeData(c, http.StatusBadRequest, 400, nil, "failed to insert metadata: "+err.Error())- return- }-- common.SuccessWithData(c, result, "success")-}-// ListTenantMembers lists all non-owner members of a tenant.// @Summary List tenant members// @Tags tenantsdiff --git a/internal/router/router.go b/internal/router/router.goindex c56c9c9e6f..51c647d4cc 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -292,12 +292,10 @@ func (r *Router) Setup(engine *gin.Engine) {tenant := v1.Group("/tenant"){tenant.GET("/list", r.tenantHandler.TenantList)- tenant.POST("/chunk_store", r.tenantHandler.CreateChunkStore) // Internal API only for GO- tenant.DELETE("/chunk_store", r.tenantHandler.DeleteChunkStore) // Internal API only for GO- tenant.POST("/metadata_store", r.tenantHandler.CreateMetadataStore) // Internal API only for GO- tenant.DELETE("/metadata_store", r.tenantHandler.DeleteMetadataStore) // Internal API only for GO- tenant.POST("/dev_insert_chunks_from_file", r.tenantHandler.InsertChunksFromFile) // Internal API only for GO- tenant.POST("/dev_insert_metadata_from_file", r.tenantHandler.InsertMetadataFromFile) // Internal API only for GO+ tenant.POST("/chunk_store", r.tenantHandler.CreateChunkStore) // Internal API only for GO+ tenant.DELETE("/chunk_store", r.tenantHandler.DeleteChunkStore) // Internal API only for GO+ tenant.POST("/metadata_store", r.tenantHandler.CreateMetadataStore) // Internal API only for GO+ tenant.DELETE("/metadata_store", r.tenantHandler.DeleteMetadataStore) // Internal API only for GO}// Document routes