Peach: Local-first video metadata scraper and media manager.
feat(library): 持续报告扫描发现的文件数与大小
longmeidaoadc5a009b2c8
diff --git a/src/peach/library_processing.py b/src/peach/library_processing.pyindex 1a900663d..ebd54666b 100644--- a/src/peach/library_processing.py+++ b/src/peach/library_processing.py@@ -1651,6 +1651,7 @@ class _SourceScan:online_roots: list[Path]scanned: int = 0probed: int = 0+ scanned_bytes: int = 0def _scan_sources(config, db_path, *, locations, asset_ids, stage, guard, update, issue):@@ -1672,8 +1673,13 @@ def _scan_sources(config, db_path, *, locations, asset_ids, stage, guard, updateresult.online_roots.append(translated_root)if scan:registered = scan_location(db_path, source, root, declared_roots=config.locations,- mounts=mounts, report=lambda line: update(stage='扫描文件'))+ mounts=mounts, report=lambda line: None,+ progress=lambda files, size: update(+ stage='扫描文件', scanned=result.scanned + files,+ scanned_bytes=result.scanned_bytes + size,+ checked=0, total=0, current_action='scanning_files'))result.scanned += registered.files+ result.scanned_bytes += registered.total_bytesresult.probed += probe_unmeasured(db_path, ffprobe, source, root, config=config,report=lambda done, total: update(stage='读取时长与分辨率', checked=done, total=total))@@ -1949,7 +1955,7 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='comode=speed_mode)):state = dict(job_id=identity, status='running',stage='读取本地资料' if retrying else '扫描文件',- checked=0, total=0, scanned=0, probed=0, identified=0, candidates=0, covers=0, previews=0,+ checked=0, total=0, scanned=0, scanned_bytes=0, probed=0, identified=0, candidates=0, covers=0, previews=0,requested_stage=stage, asset_ids=scope_ids,issue_count=0, paused_count=0, issue_preview=[], issues_truncated=False,notes={}, retryable_asset_ids=[],@@ -2020,7 +2026,7 @@ def process_library(config, db_path, candidate_root, cover_root, *, location='colocations = config.locations if location == 'configured' else {location: config.locations[location]}sources = _scan_sources(config, db_path, locations=locations, asset_ids=scope_ids,stage=stage, guard=guard, update=update, issue=issue)- state.update(scanned=sources.scanned, probed=sources.probed)+ state.update(scanned=sources.scanned, scanned_bytes=sources.scanned_bytes, probed=sources.probed)if stage == SCAN_STAGE:# `checked`/`total` 在探时长时借给进度条用过;只扫描这一段没有采集,读数归零。update(status='failed' if state['issue_count'] else 'complete',diff --git a/src/peach/scan.py b/src/peach/scan.pyindex 2fbc4f3d3..3824e4a82 100644--- a/src/peach/scan.py+++ b/src/peach/scan.py@@ -372,10 +372,24 @@ def _walk(top: Path):yield directory, sorted(files, key=lambda entry: entry.name)+class _ScanProgress:+ """结构化扫描进度按时间节流,首项与最终汇总始终报告。"""+ def __init__(self, callback):+ self.callback = callback+ self.reported_at = 0.0++ def __call__(self, files, total_bytes, *, force=False):+ now = time.monotonic()+ if force or files == 1 or now - self.reported_at >= 0.5:+ self.callback(files, total_bytes)+ self.reported_at = now++def scan_location(db_path: str | os.PathLike[str], location: str, root: str, *,declared_roots: Mapping[str, str], mounts: Mapping[str, str | Path] | None = None,report: Callable[[str], None] = print, windows: bool | None = None,+ progress: Callable[[int, int], None] = lambda files, total_bytes: None,) -> ScanResult:"""遍历 `root`(账本口径)对应的本机目录,把文件元数据 upsert 进 `asset`。@@ -388,6 +402,7 @@ def scan_location(ledger_root = PureWindowsPath(root)now = time.strftime("%Y-%m-%d %H:%M:%S")started = time.time()+ progress = _ScanProgress(progress)files = 0total = 0skipped = 0@@ -428,6 +443,7 @@ def scan_location(stat.st_size, mtime, now, now))files += 1total += stat.st_size+ progress(files, total)if len(batch) >= BATCH_SIZE:connection.executemany(_UPSERT, batch)connection.commit()@@ -456,5 +472,6 @@ def scan_location(connection.close()result = ScanResult(location, root, files, total, time.time() - started, gone,tracks, orphans, skipped, new_ids, reattached, acquired)+ progress(files, total, force=True)report(result.summary())return resultdiff --git a/tests/test_processing_control.py b/tests/test_processing_control.pyindex b30067500..2004bd2cd 100644--- a/tests/test_processing_control.py+++ b/tests/test_processing_control.py@@ -280,3 +280,26 @@ class ProcessingControlTests(unittest.TestCase):self.assertEqual(second.snapshot()['status'], 'complete')self.assertEqual(second.snapshot()['run_id'], initial['run_id'])self.assertEqual(runs.get(initial['run_id']).status, 'succeeded')++ def test_scan_progress_accumulates_discovered_counts_across_roots(self):+ from peach.library_processing import _scan_sources+ reports = []+ config = PeachConfig(self.root, self.root / 'config.toml', present=True,+ locations={'local': ('R:\\Media', 'R:\\Other')})+ def scan(*_args, **kwargs):+ kwargs['progress'](1, 10)+ kwargs['progress'](2, 30)+ return SimpleNamespace(files=2, total_bytes=30, new_ids=[])+ with patch('peach.library_processing.scan_location', side_effect=scan), \+ patch('peach.library_processing._online_source_root', return_value=self.root), \+ patch('peach.library_processing._ffprobe_path', return_value=None), \+ patch('peach.library_processing.probe_unmeasured', return_value=0), \+ patch('peach.library_processing.reattach_in'):+ result = _scan_sources(config, self.db, locations=config.locations, asset_ids=None,+ stage='scan', guard=Mock(), update=lambda **values: reports.append(values),+ issue=lambda *_args, **_kwargs: None)+ discovering = [row for row in reports if row.get('current_action') == 'scanning_files']+ self.assertEqual([row['scanned'] for row in discovering], [1, 2, 3, 4])+ self.assertEqual([row['scanned_bytes'] for row in discovering], [10, 30, 40, 60])+ self.assertTrue(all(row['total'] == 0 for row in discovering))+ self.assertEqual((result.scanned, result.scanned_bytes), (4, 60))diff --git a/tests/test_scan.py b/tests/test_scan.pyindex 6991f8c56..039dd5eda 100644--- a/tests/test_scan.py+++ b/tests/test_scan.py@@ -104,6 +104,15 @@ class WindowsShapeTests(_ScanCase):class PosixShapeTests(_ScanCase):"""macOS:账本里写 `R:\\media\\...`,遍历的是挂载点,读取侧按挂载表翻回去。"""+ def test_structured_progress_reports_discovered_files_and_bytes(self):+ progress = []+ result, _ = self._scan('R:\\media', declared_roots={'local': ('R:\\media',)},+ mounts={'local': (str(self.media),)}, windows=False,+ progress=lambda files, size: progress.append((files, size)))+ self.assertEqual(progress[0][0], 1)+ self.assertEqual(progress[-1], (3, 14))+ self.assertEqual(progress[-1], (result.files, result.total_bytes))+DECLARED = {"local": (r"R:\media",)}def test_rows_keep_the_ledger_shape_and_translate_back_to_real_files(self):