Skip to content

🎧 Your Personal Streaming Service

fix(scanner): execute GetFolderUpdateInfo in batches to avoid "Expression tree is too large (maximum depth 1000)"

Deluancde5992c4632
diff --git a/persistence/folder_repository.go b/persistence/folder_repository.go
index a586746a0..f80cbde65 100644
--- a/persistence/folder_repository.go
+++ b/persistence/folder_repository.go
@@ -95,45 +95,82 @@ func (r folderRepository) CountAll(opt ...model.QueryOptions) (int64, error) {
}
func (r folderRepository) GetFolderUpdateInfo(lib model.Library, targetPaths ...string) (map[string]model.FolderUpdateInfo, error) {
+ // If no specific paths, return all folders in the library
+ if len(targetPaths) == 0 {
+ return r.getFolderUpdateInfoAll(lib)
+ }
+
+ // Check if any path is root (return all folders)
+ for _, targetPath := range targetPaths {
+ if targetPath == "" || targetPath == "." {
+ return r.getFolderUpdateInfoAll(lib)
+ }
+ }
+
+ // Process paths in batches to avoid SQLite's expression tree depth limit (max 1000).
+ // Each path generates ~3 conditions, so batch size of 100 keeps us well under the limit.
+ const batchSize = 100
+ result := make(map[string]model.FolderUpdateInfo)
+
+ for batch := range slices.Chunk(targetPaths, batchSize) {
+ batchResult, err := r.getFolderUpdateInfoBatch(lib, batch)
+ if err != nil {
+ return nil, err
+ }
+ for id, info := range batchResult {
+ result[id] = info
+ }
+ }
+
+ return result, nil
+}
+
+// getFolderUpdateInfoAll returns update info for all non-missing folders in the library
+func (r folderRepository) getFolderUpdateInfoAll(lib model.Library) (map[string]model.FolderUpdateInfo, error) {
where := And{
Eq{"library_id": lib.ID},
Eq{"missing": false},
}
+ return r.queryFolderUpdateInfo(where)
+}
- // If specific paths are requested, include those folders and all their descendants
- if len(targetPaths) > 0 {
- // Collect folder IDs for exact target folders and path conditions for descendants
- folderIDs := make([]string, 0, len(targetPaths))
- pathConditions := make(Or, 0, len(targetPaths)*2)
+// getFolderUpdateInfoBatch returns update info for a batch of target paths and their descendants
+func (r folderRepository) getFolderUpdateInfoBatch(lib model.Library, targetPaths []string) (map[string]model.FolderUpdateInfo, error) {
+ where := And{
+ Eq{"library_id": lib.ID},
+ Eq{"missing": false},
+ }
- for _, targetPath := range targetPaths {
- if targetPath == "" || targetPath == "." {
- // Root path - include everything in this library
- pathConditions = Or{}
- folderIDs = nil
- break
- }
- // Clean the path to normalize it. Paths stored in the folder table do not have leading/trailing slashes.
- cleanPath := strings.TrimPrefix(targetPath, string(os.PathSeparator))
- cleanPath = filepath.Clean(cleanPath)
+ // Collect folder IDs for exact target folders and path conditions for descendants
+ folderIDs := make([]string, 0, len(targetPaths))
+ pathConditions := make(Or, 0, len(targetPaths)*2)
- // Include the target folder itself by ID
- folderIDs = append(folderIDs, model.FolderID(lib, cleanPath))
+ for _, targetPath := range targetPaths {
+ // Clean the path to normalize it. Paths stored in the folder table do not have leading/trailing slashes.
+ cleanPath := strings.TrimPrefix(targetPath, string(os.PathSeparator))
+ cleanPath = filepath.Clean(cleanPath)
- // Include all descendants: folders whose path field equals or starts with the target path
- // Note: Folder.Path is the directory path, so children have path = targetPath
- pathConditions = append(pathConditions, Eq{"path": cleanPath})
- pathConditions = append(pathConditions, Like{"path": cleanPath + "/%"})
- }
+ // Include the target folder itself by ID
+ folderIDs = append(folderIDs, model.FolderID(lib, cleanPath))
- // Combine conditions: exact folder IDs OR descendant path patterns
- if len(folderIDs) > 0 {
- where = append(where, Or{Eq{"id": folderIDs}, pathConditions})
- } else if len(pathConditions) > 0 {
- where = append(where, pathConditions)
- }
+ // Include all descendants: folders whose path field equals or starts with the target path
+ // Note: Folder.Path is the directory path, so children have path = targetPath
+ pathConditions = append(pathConditions, Eq{"path": cleanPath})
+ pathConditions = append(pathConditions, Like{"path": cleanPath + "/%"})
+ }
+
+ // Combine conditions: exact folder IDs OR descendant path patterns
+ if len(folderIDs) > 0 {
+ where = append(where, Or{Eq{"id": folderIDs}, pathConditions})
+ } else if len(pathConditions) > 0 {
+ where = append(where, pathConditions)
}
+ return r.queryFolderUpdateInfo(where)
+}
+
+// queryFolderUpdateInfo executes the query and returns the result map
+func (r folderRepository) queryFolderUpdateInfo(where And) (map[string]model.FolderUpdateInfo, error) {
sq := r.newSelect().Columns("id", "updated_at", "hash").Where(where)
var res []struct {
ID string