RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix: search files recursively within current folder subtree (#17779)
euvrefdb6b5fdd2a9
diff --git a/api/db/services/file_service.py b/api/db/services/file_service.pyindex ff432dfecd..6ccd57b523 100644--- a/api/db/services/file_service.py+++ b/api/db/services/file_service.py@@ -19,9 +19,10 @@ import loggingimport reimport sysimport time+from collections import dequefrom concurrent.futures import ThreadPoolExecutorfrom pathlib import Path-from typing import Union+from typing import ClassVarlogger = logging.getLogger(__name__)@@ -34,14 +35,14 @@ from api.db.services import duplicate_namefrom api.db.services.common_service import CommonServicefrom api.db.services.document_service import DocumentServicefrom api.db.services.file2document_service import File2DocumentService-from common.misc_utils import get_uuid-from common.ssrf_guard import assert_url_is_safe-from common.constants import TaskStatus, FileSource, ParserType, MAXIMUM_PAGE_NUMBERfrom api.db.services.knowledgebase_service import KnowledgebaseServicefrom api.db.services.task_service import TaskService-from api.utils.file_utils import filename_type, read_potential_broken_pdf, thumbnail_img, sanitize_path-from rag.llm.cv_model import GptV4+from api.utils.file_utils import filename_type, read_potential_broken_pdf, sanitize_path, thumbnail_imgfrom common import settings+from common.constants import MAXIMUM_PAGE_NUMBER, FileSource, ParserType, TaskStatus+from common.misc_utils import get_uuid+from common.ssrf_guard import assert_url_is_safe+from rag.llm.cv_model import GptV4class FileService(CommonService):@@ -63,7 +64,12 @@ class FileService(CommonService):# Returns:# Tuple of (file_list, total_count)if keywords:- files = cls.model.select().where((cls.model.tenant_id == tenant_id), (cls.model.parent_id == pf_id), (fn.LOWER(cls.model.name).contains(keywords.lower())), ~(cls.model.id == pf_id))+ # Keyword search covers the whole subtree under pf_id so files and+ # folders nested in sub-folders can be found too.+ subtree_ids = cls.get_subtree_ids(tenant_id, pf_id)+ files = cls.model.select().where(+ (cls.model.tenant_id == tenant_id), (cls.model.parent_id.in_(subtree_ids)), (fn.LOWER(cls.model.name).contains(keywords.lower())), ~(cls.model.id == pf_id)+ )else:files = cls.model.select().where((cls.model.tenant_id == tenant_id), (cls.model.parent_id == pf_id), ~(cls.model.id == pf_id))count = files.count()@@ -104,6 +110,29 @@ class FileService(CommonService):return res_files, count+ @classmethod+ @DB.connection_context()+ def get_subtree_ids(cls, tenant_id, pf_id):+ # Return pf_id itself plus the IDs of all entries nested under it+ # (folders and files), used to scope recursive keyword searches.+ rows = list(cls.model.select(cls.model.id, cls.model.parent_id).where(cls.model.tenant_id == tenant_id).dicts())+ children = {}+ for row in rows:+ children.setdefault(row["parent_id"], []).append(row["id"])++ ids = [pf_id]+ in_tree = {pf_id}+ queue = deque([pf_id])+ while queue:+ current = queue.popleft()+ for child in children.get(current, []):+ if child in in_tree:+ continue+ in_tree.add(child)+ ids.append(child)+ queue.append(child)+ return ids+@classmethod@DB.connection_context()def get_kb_id_by_file_id(cls, file_id):@@ -440,7 +469,6 @@ class FileService(CommonService):@classmethod@DB.connection_context()def delete(cls, file):- #return cls.delete_by_id(file.id)@classmethod@@ -457,7 +485,7 @@ class FileService(CommonService):cls.delete_folder_by_pf_id(user_id, file.id)return (cls.model.delete().where((cls.model.tenant_id == user_id) & (cls.model.id == folder_id)).execute(),)except Exception:- logging.exception("delete_folder_by_pf_id")+ logger.exception("delete_folder_by_pf_id")raise RuntimeError("Database error (File retrieval)!")@classmethod@@ -498,7 +526,7 @@ class FileService(CommonService):"source_type": FileSource.KNOWLEDGEBASE,}cls.save(**file)- File2DocumentService.save(**{"id": get_uuid(), "file_id": file["id"], "document_id": doc["id"]})+ File2DocumentService.save(id=get_uuid(), file_id=file["id"], document_id=doc["id"])@classmethod@DB.connection_context()@@ -506,7 +534,7 @@ class FileService(CommonService):try:cls.filter_update((cls.model.id << file_ids,), {"parent_id": folder_id})except Exception:- logging.exception("move_file")+ logger.exception("move_file")raise RuntimeError("Database error (File move)!")@classmethod@@ -534,7 +562,7 @@ class FileService(CommonService):if e:try:if str(doc.kb_id) != str(kb.id):- logging.warning(+ logger.warning("Existing document id collision detected for %s: belongs to kb_id=%s, incoming kb_id=%s. Skipping update to avoid cross-KB overwrite.",doc_id,doc.kb_id,@@ -559,7 +587,7 @@ class FileService(CommonService):if new_hash != old_hash:files.append((doc, blob))except Exception as exc:- logging.exception(f"Failed to update document {doc_id}: {exc}")+ logger.exception("Failed to update document %s", doc_id)err.append(file.filename + ": " + str(exc))continuetry:@@ -605,7 +633,7 @@ class FileService(CommonService):FileService.add_file_from_kb(doc, kb_folder["id"], kb.tenant_id)files.append((doc, blob))- except Exception as e:+ except Exception as e: # noqa: BLE001 - collect per-file errors and keep processing the resterr.append(file.filename + ": " + str(e))return err, files@@ -617,7 +645,7 @@ class FileService(CommonService):files = cls.model.select().where((cls.model.parent_id == parent_id) & (cls.model.id != parent_id))return list(files)except Exception:- logging.exception("list_by_parent_id failed")+ logger.exception("list_by_parent_id failed")raise RuntimeError("Database error (list_by_parent_id)!")@staticmethod@@ -635,8 +663,8 @@ class FileService(CommonService):@staticmethoddef parse(filename, blob, img_base64=True, tenant_id=None, layout_recognize=None):- from rag.app import audio, email, naive, picture, presentationfrom api.apps import current_user+ from rag.app import audio, email, naive, picture, presentationdef dummy(prog=None, msg=""):pass@@ -684,16 +712,16 @@ class FileService(CommonService):try:e, doc = DocumentService.get_by_id(doc_id)if not e:- raise Exception("document not found")+ raise RuntimeError("document not found")tenant_id = DocumentService.get_tenant_id(doc_id)if not tenant_id:- raise Exception("Tenant not found!")+ raise RuntimeError("Tenant not found!")b, n = File2DocumentService.get_storage_address(doc_id=doc_id)TaskService.filter_delete([Task.doc_id == doc_id])if not DocumentService.remove_document(doc, tenant_id):- raise Exception("Database error (Document removal)!")+ raise RuntimeError("Database error (Document removal)!")f2d = File2DocumentService.get_by_document_id(doc_id)deleted_file_count = 0@@ -712,12 +740,12 @@ class FileService(CommonService):kb_table_num_map[kb_id] -= 1if kb_table_num_map[kb_id] <= 0:KnowledgebaseService.delete_field_map(kb_id)- except Exception as e:+ except Exception as e: # noqa: BLE001 - aggregate per-document errors and continue deleting the resterrors += str(e)return errors- _ALLOWED_SCHEMES = {"http", "https"}+ _ALLOWED_SCHEMES: ClassVar[set[str]] = {"http", "https"}@staticmethoddef _validate_url_for_crawl(url: str) -> tuple[str, str]:@@ -760,8 +788,10 @@ class FileService(CommonService):}if url:- import requests as _requestsfrom urllib.parse import urljoin as _urljoin++ import requests as _requests+from api.utils.web_utils import BROWSER_FETCH_TIMEOUT, browser_fetch_slot_MAX_CRAWL_REDIRECTS = 10@@ -807,7 +837,7 @@ class FileService(CommonService):# skipping DNS entirely and eliminating the rebinding window._map_rules = ",".join(f"MAP {h} {ip}" for h, ip in host_pins.items())- from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, DefaultMarkdownGenerator, PruningContentFilter, CrawlResult+ from crawl4ai import AsyncWebCrawler, BrowserConfig, CrawlerRunConfig, CrawlResult, DefaultMarkdownGenerator, PruningContentFilterfilename = re.sub(r"\?.*", "", url.split("/")[-1])@@ -836,7 +866,7 @@ class FileService(CommonService):return structured(file.filename, filename_type(file.filename), file.read(), file.content_type)@staticmethod- def get_files(files: Union[None, list[dict]], raw: bool = False, layout_recognize: str = None) -> Union[list[str], tuple[list[str], list[dict]]]:+ def get_files(files: None | list[dict], raw: bool = False, layout_recognize: str | None = None) -> list[str] | tuple[list[str], list[dict]]:if not files:return []diff --git a/internal/dao/file.go b/internal/dao/file.goindex 9d0ddb71cd..7f92ec61e1 100644--- a/internal/dao/file.go+++ b/internal/dao/file.go@@ -45,17 +45,26 @@ func (dao *FileDAO) GetByID(ctx context.Context, db *gorm.DB, id string) (*entitreturn &file, nil}-// GetByPfID gets files by parent folder ID with pagination and filtering+// GetByPfID gets files by parent folder ID with pagination and filtering.+// When keywords is empty, only direct children of pfID are listed; when+// keywords is non-empty, the search covers the whole subtree under pfID so+// files and folders nested in sub-folders can be found too.func (dao *FileDAO) GetByPfID(ctx context.Context, db *gorm.DB, tenantID, pfID string, page, pageSize int, orderBy string, desc bool, keywords string) ([]*entity.File, int64, error) {var files []*entity.Filevar total int64query := db.WithContext(ctx).Model(&entity.File{}).- Where("tenant_id = ? AND parent_id = ? AND id != ?", tenantID, pfID, pfID)+ Where("tenant_id = ? AND id != ?", tenantID, pfID)- // Apply keyword filterif keywords != "" {- query = query.Where("LOWER(name) LIKE ?", "%"+strings.ToLower(keywords)+"%")+ descendantIDs, err := dao.GetSubtreeIDs(ctx, db, tenantID, pfID)+ if err != nil {+ return nil, 0, err+ }+ query = query.Where("parent_id IN ?", descendantIDs).+ Where("LOWER(name) LIKE ?", "%"+strings.ToLower(keywords)+"%")+ } else {+ query = query.Where("parent_id = ?", pfID)}// Count total@@ -85,6 +94,43 @@ func (dao *FileDAO) GetByPfID(ctx context.Context, db *gorm.DB, tenantID, pfID sreturn files, total, nil}+// GetSubtreeIDs returns pfID itself plus the IDs of all entries nested under+// it (folders and files), used to scope recursive keyword searches.+func (dao *FileDAO) GetSubtreeIDs(ctx context.Context, db *gorm.DB, tenantID, pfID string) ([]string, error) {+ var rows []struct {+ ID string+ ParentID string+ }+ if err := db.WithContext(ctx).Model(&entity.File{}).+ Select("id", "parent_id").+ Where("tenant_id = ?", tenantID).+ Find(&rows).Error; err != nil {+ return nil, err+ }++ children := make(map[string][]string, len(rows))+ for _, row := range rows {+ children[row.ParentID] = append(children[row.ParentID], row.ID)+ }++ ids := []string{pfID}+ inTree := map[string]struct{}{pfID: {}}+ queue := []string{pfID}+ for len(queue) > 0 {+ cur := queue[0]+ queue = queue[1:]+ for _, child := range children[cur] {+ if _, ok := inTree[child]; ok {+ continue+ }+ inTree[child] = struct{}{}+ ids = append(ids, child)+ queue = append(queue, child)+ }+ }+ return ids, nil+}+// GetRootFolder gets or creates root folder for tenantfunc (dao *FileDAO) GetRootFolder(ctx context.Context, db *gorm.DB, tenantID string) (*entity.File, error) {var file entity.Filediff --git a/internal/dao/file_test.go b/internal/dao/file_test.gonew file mode 100644index 0000000000..fd797e5bfe--- /dev/null+++ b/internal/dao/file_test.go@@ -0,0 +1,150 @@+//+// Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.+//++package dao++import (+ "context"+ "testing"++ "github.com/glebarez/sqlite"+ "gorm.io/gorm"++ "ragflow/internal/entity"+)++func setupFileTestDB(t *testing.T) *gorm.DB {+ t.Helper()++ db, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{+ TranslateError: true,+ })+ if err != nil {+ t.Fatalf("failed to open sqlite: %v", err)+ }++ if err := db.AutoMigrate(&entity.File{}); err != nil {+ t.Fatalf("failed to migrate: %v", err)+ }++ return db+}++func testFile(t *testing.T, db *gorm.DB, id, parentID, tenantID, name, fileType string) {+ t.Helper()+ f := &entity.File{+ ID: id,+ ParentID: parentID,+ TenantID: tenantID,+ CreatedBy: tenantID,+ Name: name,+ Type: fileType,+ }+ if err := db.Create(f).Error; err != nil {+ t.Fatalf("failed to create file %s: %v", id, err)+ }+}++// seedFileTree builds: root -> {dirA -> {subB -> [notes-deep.txt]}, top-report.pdf}, other -> [outside-report.txt]+func seedFileTree(t *testing.T, db *gorm.DB) {+ t.Helper()+ testFile(t, db, "root", "root", "t1", "/", "folder")+ testFile(t, db, "dirA", "root", "t1", "dirA", "folder")+ testFile(t, db, "subB", "dirA", "t1", "subB", "folder")+ testFile(t, db, "f-deep", "subB", "t1", "notes-deep.txt", "doc")+ testFile(t, db, "f-top", "root", "t1", "top-report.pdf", "doc")+ testFile(t, db, "other", "other", "t1", "other", "folder")+ testFile(t, db, "f-out", "other", "t1", "outside-report.txt", "doc")+ testFile(t, db, "f-t2", "root", "t2", "report-t2.txt", "doc")+}++func TestFileDAO_GetByPfID_KeywordsSearchesSubtree(t *testing.T) {+ db := setupFileTestDB(t)+ seedFileTree(t, db)+ d := NewFileDAO()+ ctx := context.Background()++ files, total, err := d.GetByPfID(ctx, db, "t1", "root", 1, 15, "create_time", true, "report")+ if err != nil {+ t.Fatalf("GetByPfID failed: %v", err)+ }+ if total != 1 || len(files) != 1 || files[0].ID != "f-top" {+ t.Fatalf("expected only f-top, got total=%d files=%v", total, files)+ }++ // Nested file two levels down must be found from the root folder.+ files, total, err = d.GetByPfID(ctx, db, "t1", "root", 1, 15, "create_time", true, "notes")+ if err != nil {+ t.Fatalf("GetByPfID failed: %v", err)+ }+ if total != 1 || len(files) != 1 || files[0].ID != "f-deep" {+ t.Fatalf("expected nested f-deep, got total=%d files=%v", total, files)+ }++ // Folders themselves are searchable by name.+ files, total, err = d.GetByPfID(ctx, db, "t1", "root", 1, 15, "create_time", true, "sub")+ if err != nil {+ t.Fatalf("GetByPfID failed: %v", err)+ }+ if total != 1 || len(files) != 1 || files[0].ID != "subB" {+ t.Fatalf("expected folder subB, got total=%d files=%v", total, files)+ }+}++func TestFileDAO_GetByPfID_KeywordsScopedToSubtree(t *testing.T) {+ db := setupFileTestDB(t)+ seedFileTree(t, db)+ d := NewFileDAO()+ ctx := context.Background()++ // Searching inside dirA must not match files outside that subtree.+ files, total, err := d.GetByPfID(ctx, db, "t1", "dirA", 1, 15, "create_time", true, "report")+ if err != nil {+ t.Fatalf("GetByPfID failed: %v", err)+ }+ if total != 0 || len(files) != 0 {+ t.Fatalf("expected no results outside subtree, got total=%d files=%v", total, files)+ }++ // Tenant isolation still applies.+ files, total, err = d.GetByPfID(ctx, db, "t2", "root", 1, 15, "create_time", true, "report")+ if err != nil {+ t.Fatalf("GetByPfID failed: %v", err)+ }+ if total != 1 || len(files) != 1 || files[0].ID != "f-t2" {+ t.Fatalf("expected only tenant t2 file, got total=%d files=%v", total, files)+ }+}++func TestFileDAO_GetByPfID_NoKeywordsListsDirectChildren(t *testing.T) {+ db := setupFileTestDB(t)+ seedFileTree(t, db)+ d := NewFileDAO()+ ctx := context.Background()++ files, total, err := d.GetByPfID(ctx, db, "t1", "root", 1, 15, "create_time", true, "")+ if err != nil {+ t.Fatalf("GetByPfID failed: %v", err)+ }+ if total != 2 || len(files) != 2 {+ t.Fatalf("expected 2 direct children, got total=%d files=%v", total, files)+ }+ for _, f := range files {+ if f.ParentID != "root" || f.ID == "root" {+ t.Fatalf("unexpected entry in direct listing: %+v", f)+ }+ }+}