Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(processing): 补全链路批量查询使用单参数集合

zouyonghe9b2e4eb0b585
diff --git a/src/peach/library_processing.py b/src/peach/library_processing.py
index 30e359e01..e801c69ec 100644
--- a/src/peach/library_processing.py
+++ b/src/peach/library_processing.py
@@ -1942,11 +1942,10 @@ def _chosen_asset_paths(db_path, asset_ids):
"""
if not asset_ids:
return {}
- placeholders = ','.join('?' * len(asset_ids))
with closing(sqlite3.connect(db_path, timeout=30)) as connection:
rows = connection.execute(
- f"SELECT location,path FROM asset WHERE id IN ({placeholders}) "
- "AND medium='video' AND disposal IS NULL", asset_ids).fetchall()
+ "SELECT location,path FROM asset WHERE id IN (SELECT value FROM json_each(?)) "
+ "AND medium='video' AND disposal IS NULL", (json.dumps(asset_ids),)).fetchall()
chosen = {}
for source, path in rows:
chosen.setdefault(str(source), []).append(translate_ledger_path(path))
@@ -2156,8 +2155,8 @@ def _collection_rows(db_path, *, locations, online_roots, asset_ids):
query = f"SELECT asset.*, {multi} FROM asset WHERE medium='video' AND disposal IS NULL"
parameters = []
if asset_ids is not None:
- query += ' AND id IN (' + ','.join('?' * len(asset_ids)) + ')'
- parameters = asset_ids
+ query += ' AND id IN (SELECT value FROM json_each(?))'
+ parameters = [json.dumps(asset_ids)]
query += ' ORDER BY id'
library_roots = tuple(root for roots in locations.values() for root in roots)
with closing(sqlite3.connect(db_path, timeout=30)) as connection:
diff --git a/src/peach/processing_control.py b/src/peach/processing_control.py
index c88bd9445..360b321d0 100644
--- a/src/peach/processing_control.py
+++ b/src/peach/processing_control.py
@@ -336,8 +336,8 @@ def initial_estimate(config, db_path, stage, asset_ids=None, mode=None):
query = "SELECT COUNT(*), COALESCE(SUM(size),0) FROM asset WHERE medium='video' AND disposal IS NULL"
parameters = []
if asset_ids is not None:
- query += ' AND id IN (' + ','.join('?' * len(asset_ids)) + ')'
- parameters += asset_ids
+ query += ' AND id IN (SELECT value FROM json_each(?))'
+ parameters.append(json.dumps(asset_ids))
if stage == 'previews':
query += " AND location='local' AND (snapshot_path IS NULL OR snapshot_path='')"
if config.locations:
diff --git a/src/peach/processing_traffic_plan.py b/src/peach/processing_traffic_plan.py
index e1b98d0f0..61bf5d062 100644
--- a/src/peach/processing_traffic_plan.py
+++ b/src/peach/processing_traffic_plan.py
@@ -2,6 +2,7 @@
from __future__ import annotations
from itertools import islice
+import json
import os
import sqlite3
from statistics import median
@@ -28,8 +29,8 @@ def ledger_rows(db_path, asset_ids=None, *, library_roots=()):
FROM asset WHERE medium='video' AND disposal IS NULL"""
parameters = []
if asset_ids is not None:
- query += ' AND id IN (' + ','.join('?' for _ in asset_ids) + ')'
- parameters = asset_ids
+ query += ' AND id IN (SELECT value FROM json_each(?))'
+ parameters = [json.dumps(asset_ids)]
with closing(sqlite3.connect(db_path, timeout=30)) as connection:
connection.row_factory = sqlite3.Row
return [dict(row) for row in connection.execute(query, parameters)
diff --git a/tests/test_metadata_library.py b/tests/test_metadata_library.py
index 796dcbade..57fed9caa 100644
--- a/tests/test_metadata_library.py
+++ b/tests/test_metadata_library.py
@@ -3529,6 +3529,30 @@ class LibraryPreviewCompletionTests(unittest.TestCase):
with self.subTest(invalid=invalid), self.assertRaises(ValueError):
_asset_selection(self.db, [2, 1, invalid])
+ def test_selected_processing_queries_fit_the_connection_parameter_limit(self):
+ from peach.library_processing import _chosen_asset_paths, _collection_rows
+ from peach.processing_control import initial_estimate
+ from peach.processing_traffic_plan import ledger_rows
+ connect = sqlite3.connect
+ def limited_connection(*args, **kwargs):
+ connection = connect(*args, **kwargs)
+ connection.setlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER, 3)
+ return connection
+ selection = [2, 1] * 4
+ with patch('peach.library_processing.sqlite3.connect', side_effect=limited_connection):
+ with self.subTest(query='paths'):
+ self.assertEqual(set(_chosen_asset_paths(self.db, selection)), {'local', 'pikpak'})
+ with self.subTest(query='collection'):
+ rows, _ = _collection_rows(self.db, locations=self.config.locations,
+ online_roots=(self.media,), asset_ids=selection)
+ self.assertEqual([row['id'] for row in rows], [1])
+ with self.subTest(query='estimate'):
+ estimate = initial_estimate(self.config, self.db, 'all', selection, mode='high')
+ self.assertEqual(estimate['estimate_file_count'], 2)
+ with self.subTest(query='traffic'):
+ self.assertEqual({row['id'] for row in ledger_rows(self.db, selection)}, {1, 2})
+ self.assertEqual(ledger_rows(self.db, []), [])
+
def test_selected_assets_require_active_videos(self):
for selection in ([], [True], [0], [-1], ['1'], '1', [99], [3], [4]):
with self.subTest(selection=selection), self.assertRaises(ValueError), \