Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

feat(library): 持续报告扫描发现的文件数与大小

longmeidaoadc5a009b2c8
diff --git a/src/peach/library_processing.py b/src/peach/library_processing.py
index 1a900663d..ebd54666b 100644
--- a/src/peach/library_processing.py
+++ b/src/peach/library_processing.py
@@ -1651,6 +1651,7 @@ class _SourceScan:
online_roots: list[Path]
scanned: int = 0
probed: int = 0
+ scanned_bytes: int = 0
def _scan_sources(config, db_path, *, locations, asset_ids, stage, guard, update, issue):
@@ -1672,8 +1673,13 @@ def _scan_sources(config, db_path, *, locations, asset_ids, stage, guard, update
result.online_roots.append(translated_root)
if scan:
registered = scan_location(db_path, source, root, declared_roots=config.locations,
- mounts=mounts, report=lambda line: update(stage='扫描文件'))
+ mounts=mounts, report=lambda line: None,
+ progress=lambda files, size: update(
+ stage='扫描文件', scanned=result.scanned + files,
+ scanned_bytes=result.scanned_bytes + size,
+ checked=0, total=0, current_action='scanning_files'))
result.scanned += registered.files
+ result.scanned_bytes += registered.total_bytes
result.probed += probe_unmeasured(
db_path, ffprobe, source, root, config=config,
report=lambda done, total: update(stage='读取时长与分辨率', checked=done, total=total))
@@ -1949,7 +1955,7 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='co
mode=speed_mode)):
state = dict(job_id=identity, status='running',
stage='读取本地资料' if retrying else '扫描文件',
- checked=0, total=0, scanned=0, probed=0, identified=0, candidates=0, covers=0, previews=0,
+ checked=0, total=0, scanned=0, scanned_bytes=0, probed=0, identified=0, candidates=0, covers=0, previews=0,
requested_stage=stage, asset_ids=scope_ids,
issue_count=0, paused_count=0, issue_preview=[], issues_truncated=False,
notes={}, retryable_asset_ids=[],
@@ -2020,7 +2026,7 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='co
locations = config.locations if location == 'configured' else {location: config.locations[location]}
sources = _scan_sources(config, db_path, locations=locations, asset_ids=scope_ids,
stage=stage, guard=guard, update=update, issue=issue)
- state.update(scanned=sources.scanned, probed=sources.probed)
+ state.update(scanned=sources.scanned, scanned_bytes=sources.scanned_bytes, probed=sources.probed)
if stage == SCAN_STAGE:
# `checked`/`total` 在探时长时借给进度条用过;只扫描这一段没有采集,读数归零。
update(status='failed' if state['issue_count'] else 'complete',
diff --git a/src/peach/scan.py b/src/peach/scan.py
index 2fbc4f3d3..3824e4a82 100644
--- a/src/peach/scan.py
+++ b/src/peach/scan.py
@@ -372,10 +372,24 @@ def _walk(top: Path):
yield directory, sorted(files, key=lambda entry: entry.name)
+class _ScanProgress:
+ """结构化扫描进度按时间节流,首项与最终汇总始终报告。"""
+ def __init__(self, callback):
+ self.callback = callback
+ self.reported_at = 0.0
+
+ def __call__(self, files, total_bytes, *, force=False):
+ now = time.monotonic()
+ if force or files == 1 or now - self.reported_at >= 0.5:
+ self.callback(files, total_bytes)
+ self.reported_at = now
+
+
def scan_location(
db_path: str | os.PathLike[str], location: str, root: str, *,
declared_roots: Mapping[str, str], mounts: Mapping[str, str | Path] | None = None,
report: Callable[[str], None] = print, windows: bool | None = None,
+ progress: Callable[[int, int], None] = lambda files, total_bytes: None,
) -> ScanResult:
"""遍历 `root`(账本口径)对应的本机目录,把文件元数据 upsert 进 `asset`。
@@ -388,6 +402,7 @@ def scan_location(
ledger_root = PureWindowsPath(root)
now = time.strftime("%Y-%m-%d %H:%M:%S")
started = time.time()
+ progress = _ScanProgress(progress)
files = 0
total = 0
skipped = 0
@@ -428,6 +443,7 @@ def scan_location(
stat.st_size, mtime, now, now))
files += 1
total += stat.st_size
+ progress(files, total)
if len(batch) >= BATCH_SIZE:
connection.executemany(_UPSERT, batch)
connection.commit()
@@ -456,5 +472,6 @@ def scan_location(
connection.close()
result = ScanResult(location, root, files, total, time.time() - started, gone,
tracks, orphans, skipped, new_ids, reattached, acquired)
+ progress(files, total, force=True)
report(result.summary())
return result
diff --git a/tests/test_processing_control.py b/tests/test_processing_control.py
index b30067500..2004bd2cd 100644
--- a/tests/test_processing_control.py
+++ b/tests/test_processing_control.py
@@ -280,3 +280,26 @@ class ProcessingControlTests(unittest.TestCase):
self.assertEqual(second.snapshot()['status'], 'complete')
self.assertEqual(second.snapshot()['run_id'], initial['run_id'])
self.assertEqual(runs.get(initial['run_id']).status, 'succeeded')
+
+ def test_scan_progress_accumulates_discovered_counts_across_roots(self):
+ from peach.library_processing import _scan_sources
+ reports = []
+ config = PeachConfig(self.root, self.root / 'config.toml', present=True,
+ locations={'local': ('R:\\Media', 'R:\\Other')})
+ def scan(*_args, **kwargs):
+ kwargs['progress'](1, 10)
+ kwargs['progress'](2, 30)
+ return SimpleNamespace(files=2, total_bytes=30, new_ids=[])
+ with patch('peach.library_processing.scan_location', side_effect=scan), \
+ patch('peach.library_processing._online_source_root', return_value=self.root), \
+ patch('peach.library_processing._ffprobe_path', return_value=None), \
+ patch('peach.library_processing.probe_unmeasured', return_value=0), \
+ patch('peach.library_processing.reattach_in'):
+ result = _scan_sources(config, self.db, locations=config.locations, asset_ids=None,
+ stage='scan', guard=Mock(), update=lambda **values: reports.append(values),
+ issue=lambda *_args, **_kwargs: None)
+ discovering = [row for row in reports if row.get('current_action') == 'scanning_files']
+ self.assertEqual([row['scanned'] for row in discovering], [1, 2, 3, 4])
+ self.assertEqual([row['scanned_bytes'] for row in discovering], [10, 30, 40, 60])
+ self.assertTrue(all(row['total'] == 0 for row in discovering))
+ self.assertEqual((result.scanned, result.scanned_bytes), (4, 60))
diff --git a/tests/test_scan.py b/tests/test_scan.py
index 6991f8c56..039dd5eda 100644
--- a/tests/test_scan.py
+++ b/tests/test_scan.py
@@ -104,6 +104,15 @@ class WindowsShapeTests(_ScanCase):
class PosixShapeTests(_ScanCase):
"""macOS:账本里写 `R:\\media\\...`,遍历的是挂载点,读取侧按挂载表翻回去。"""
+ def test_structured_progress_reports_discovered_files_and_bytes(self):
+ progress = []
+ result, _ = self._scan('R:\\media', declared_roots={'local': ('R:\\media',)},
+ mounts={'local': (str(self.media),)}, windows=False,
+ progress=lambda files, size: progress.append((files, size)))
+ self.assertEqual(progress[0][0], 1)
+ self.assertEqual(progress[-1], (3, 14))
+ self.assertEqual(progress[-1], (result.files, result.total_bytes))
+
DECLARED = {"local": (r"R:\media",)}
def test_rows_keep_the_ledger_shape_and_translate_back_to_real_files(self):