Peach: Local-first video metadata scraper and media manager.
fix(discovery): 补齐系统目录边界与还原路径保护
longmeidao9d1b9e353f48
diff --git a/docs/REUSE.md b/docs/REUSE.mdindex 86a71e227..4f6422f87 100644--- a/docs/REUSE.md+++ b/docs/REUSE.md@@ -1193,5 +1193,7 @@ Python 源文件纳入 `src/peach/library_sharing.py` 与 `src/peach/library_pro- TrueNAS 的 `.zfs/snapshot` 见 [CORE 快照恢复说明](https://www.truenas.com/docs/core/13.3/coretutorials/printview/);NetApp 的 [`.snapshot`](https://docs.netapp.com/us-en/ontap/data-protection/snapshot-copies-work-concept.html) 与 [`~snapshot`](https://docs.netapp.com/us-en/ontap/smb-admin/access-snapshot-directories-clients-smb2-concept.html) 使用目录段精确匹配。Samba 仅采用 [默认 `.recycle`](https://www.samba.org/samba/docs/current/man-html/vfs_recycle.8.html),自定义回收站名称不能从名字推断。- AFP 的 `.AppleDouble`、`.AppleDesktop`、`.AppleDB`、`Network Trash Folder`、`Icon\r`、`.VolumeIcon.icns` 见 [Netatalk 专用文件与目录](https://netatalk.io/docs/Special-Files-and-Folders)。macOS 的 `.Spotlight-V100`、`.Trashes`、`.fseventsd` 可见 [Apple Remote Desktop 指南](https://images.apple.com/server/docs/ARD_3_Admin_Guide_v3.3.pdf),`__MACOSX` 中的 AppleDouble 见 [Apple DTS 取证示例](https://developer.apple.com/forums/thread/772398)。`.DS_Store` 见 [Apple SMB 说明](https://support.apple.com/102064),`.localized` 沿用已有文件名规则。- Windows 的 [`System Volume Information`](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/fltkernel/nf-fltkernel-fltcreatesystemvolumeinformationfolder)、[`$Recycle.Bin`](https://learn.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/recycle-bin-corrupted)、[`desktop.ini`](https://learn.microsoft.com/en-us/windows/win32/shell/how-to-customize-folders-with-desktop-ini) 和 [`Thumbs.db`](https://learn.microsoft.com/en-us/windows/client-management/mdm/policy-csp-admx-thumbnails) 分别按目录或文件名称匹配。+- Linux 挂载点的 `.Trash` 和 `.Trash-<UID>` 见 [FreeDesktop Trash Specification](https://specifications.freedesktop.org/trash/1.0/),UID 只匹配完整 ASCII 十进制数字;`.Trash-user` 和 `.Trash-1000-old` 仍可登记。Snapper 管理的 Btrfs 快照目录 `.snapshots` 见 [openSUSE 说明](https://doc.opensuse.org/documentation/tumbleweed/snapper/);macOS `.TemporaryItems` 见 [Apple 文件系统模板](https://github.com/apple-oss-distributions/files/blob/main/hierarchy.osx)。这些名称按完整目录段匹配,目录本身作为扫描根、恢复断点或推送路径也跳过,不进入内部文件。+- TerraMaster 支持团队确认的 [`#recycle`](https://forum.terra-master.com/en/viewtopic.php?t=2443) 已由同名规则覆盖。[ASUSTOR](https://www.asustor.com/en/online/online_help?id=11)、[fnOS](https://help.fnnas.com/articles/v1/file/team) 和 [UGREEN](https://support.ugnas.com/detail/article/en-US/159) 的已查官方说明介绍回收站功能,未据此推导物理目录名称,也不将普通 `Recycle Bin`、`Trash` 或中文文件夹名加入过滤。-不排除任意 `@` 目录或任意点开头文件;`@creator/movie.mp4`、`.hidden.mp4`、`@eaDir-old` 和普通 `Snapshots` 仍可登记。旧数据沿用 `system_file_repair` 可还原隔离,仅处理无番号、无收藏和个人记录的 `other` 行,保留文件与关联行;上一版回执已经隔离过的相同资产与路径不再处理,以保留后来的手动还原。旧视频、图片和音频不自动隔离;过滤生效不代表这些历史资产已复核。+不排除任意 `@` 目录或任意点开头文件;`@creator/movie.mp4`、`.hidden.mp4`、`@eaDir-old` 和普通 `Snapshots` 仍可登记。旧数据沿用 `system_file_repair` 可还原隔离,仅处理无番号、无收藏和个人记录的 `other` 行,保留文件与关联行;上一版回执已经隔离过的相同资产与 Windows 路径不再处理,以保留后来的手动还原。路径按 `PureWindowsPath` 比较,大小写和斜杠变化不影响保护,同一 ID 改为另一路径仍按规则检查。旧视频、图片和音频不自动隔离;过滤生效不代表这些历史资产已复核。diff --git a/src/peach/discovery_rules.py b/src/peach/discovery_rules.pyindex 72428f6cf..940b9246c 100644--- a/src/peach/discovery_rules.py+++ b/src/peach/discovery_rules.py@@ -1,14 +1,15 @@"""全量扫描、推送与下载登记共用的系统元数据判据。"""from pathlib import PureWindowsPath+import re# 厂商保留名称的来源与边界见 docs/REUSE.md「系统元数据发现过滤」。_DIRECTORIES = frozenset({'@eadir', '@tmp', '@sharebin', '#recycle', '#snapshot', '@snapshot','@recycle', '@recently-snapshot', '.@__thumb', '.@__qini', '.@upload_cache', '.streams',- '.zfs', '.snapshot', '~snapshot', '.recycle',+ '.zfs', '.snapshot', '.snapshots', '~snapshot', '.recycle', '.trash','.appledouble', '.appledesktop', '.appledb', 'network trash folder',- '.spotlight-v100', '.trashes', '.fseventsd', '__macosx',+ '.spotlight-v100', '.trashes', '.temporaryitems', '.fseventsd', '__macosx','$recycle.bin', 'system volume information',})_FILES = frozenset({@@ -18,7 +19,8 @@ _FILES = frozenset({def system_directory(name: str) -> bool:lowered = name.casefold()- return lowered in _DIRECTORIES or lowered.startswith('.wfm_upload')+ return (lowered in _DIRECTORIES or lowered.startswith('.wfm_upload')+ or re.fullmatch(r'\.trash-[0-9]+', lowered) is not None)def system_file(name: str) -> bool:@@ -29,5 +31,5 @@ def system_file(name: str) -> bool:def system_path(path: str) -> bool:parts = PureWindowsPath(path).parts- return any(system_directory(part) for part in parts[:-1]) or (+ return any(system_directory(part) for part in parts) or (bool(parts) and system_file(parts[-1]))diff --git a/src/peach/push_discovery.py b/src/peach/push_discovery.pyindex d4e5f7823..d6fefa374 100644--- a/src/peach/push_discovery.py+++ b/src/peach/push_discovery.py@@ -44,7 +44,7 @@ from typing import Callablefrom .fsutil import atomic_write_textfrom .repository import in_background-from .discovery_rules import system_directory, system_path+from .discovery_rules import system_pathLOGGER = logging.getLogger(__name__)@@ -695,7 +695,7 @@ class PushDiscoveryService:return self.queue.submit(location, path, time.monotonic())def submit_move(self, location, old, new, *, directory=False):- if ignored(new) or (directory and system_directory(PureWindowsPath(new).name)):+ if ignored(new):returnif directory:from .media_folders import path_predicatediff --git a/src/peach/scan.py b/src/peach/scan.pyindex bfdec5ad0..e6c697ea0 100644--- a/src/peach/scan.py+++ b/src/peach/scan.py@@ -482,6 +482,8 @@ def _walk(top: Path, pending: list[str] | None = None, *, onerror=lambda error:列表自带这两项,不必再对每个文件发一次 `os.stat`——网盘挂载上那一次就是一趟往返,几万个文件就是几万趟。读不了的目录跳过;符号链接指向的目录列出但不进入。"""+ if system_path(str(top)):+ returnfrom .smb_media import SmbResourceif isinstance(top, SmbResource):yield from _walk_smb(top, pending, onerror)@@ -490,6 +492,8 @@ def _walk(top: Path, pending: list[str] | None = None, *, onerror=lambda error:while stack:checkpoint('scan')directory = stack.pop()+ if system_path(directory):+ continuetry:if read_directory is not None:entries = read_directory(directory)@@ -540,6 +544,8 @@ def _walk_smb(top, pending, onerror):while stack:checkpoint('scan')directory = stack.pop()+ if system_path(directory):+ continueresource = top.joinpath(*_relative_directory(directory, top))try:try:@@ -585,6 +591,12 @@ class _ScanProgress:def scan_location(db_path, location, root, **kwargs) -> ScanResult:"""命令行按来源与根目录保存断点,组合任务可提供自己的持久检查点。"""+ if system_path(root):+ check_scan_target(location, root, declared_roots=kwargs['declared_roots'])+ result = ScanResult(location, root, 0, 0, 0, 0)+ kwargs.get('progress', lambda files, total: None)(0, 0)+ kwargs.get('report', print)(result.summary())+ return resultif 'save_checkpoint' in kwargs or 'resume_state' in kwargs:return _scan_location(db_path, location, root, **kwargs)from .update_tasks import state_fordiff --git a/src/peach/system_file_repair.py b/src/peach/system_file_repair.pyindex db6668eb0..de815e994 100644--- a/src/peach/system_file_repair.py+++ b/src/peach/system_file_repair.py@@ -1,5 +1,6 @@"""将旧推送发现登记的系统元数据放入可还原回收站。"""import json+from pathlib import PureWindowsPathfrom .discovery_rules import system_pathfrom .personal_records import record_holders@@ -8,13 +9,13 @@ PRIOR_RULE = 'discovery-system-files-1'RULE = 'discovery-system-files-2'-def _previously_isolated(connection) -> set[tuple[int, str]]:+def _previously_isolated(connection) -> set[tuple[int, PureWindowsPath]]:"""旧回执中隔离过的同一路径不再处理,保留用户后来的还原决定。"""receipts = connection.execute('SELECT json_extract(result_json,?) FROM ledger_upgrade_repair ''WHERE json_type(result_json,?) IS NOT NULL',(f'$."{PRIOR_RULE}"', f'$."{PRIOR_RULE}"'))- return {(row['id'], row['path']) for receipt in receipts+ return {(row['id'], PureWindowsPath(row['path'])) for receipt in receiptsfor row in json.loads(receipt[0])['isolated']}@@ -24,7 +25,7 @@ def collect(connection) -> list[dict]:"SELECT id,path,disposal,disposal_reason,feedback_at FROM asset ""WHERE disposal IS NULL AND medium='other' AND coalesce(code,'')='' ""AND coalesce(feedback,'')<>'like'")- if system_path(row['path']) and (row['id'], row['path']) not in previous]+ if system_path(row['path']) and (row['id'], PureWindowsPath(row['path'])) not in previous]protected = record_holders(connection, [row['id'] for row in rows])return [row for row in rows if row['id'] not in protected and not connection.execute("SELECT 1 FROM review_decision WHERE category='junk_file' AND item_key=? AND status='rejected'",diff --git a/tests/test_push_discovery.py b/tests/test_push_discovery.pyindex fde60660d..147fcd9ed 100644--- a/tests/test_push_discovery.py+++ b/tests/test_push_discovery.py@@ -211,7 +211,8 @@ class IgnoreAndSourceTests(unittest.TestCase):with self.subTest(name=repr(name)):self.assertTrue(push.ignored(str(PureWindowsPath(r'R:\media', name))))for name in ('@creator', 'normal', '@eaDir-old', '#recycle-old', '.snapshot-old',- 'Snapshots', 'Recycle Bin', 'Temporary Items'):+ 'Snapshots', 'Recycle Bin', 'Temporary Items', '.Trash-', '.Trash-1000-old',+ '.Trash-user', '.Trash-١٠٠٠', '.snapshots-old', '.TemporaryItems-old'):with self.subTest(directory=name):self.assertFalse(system_directory(name))self.assertFalse(push.ignored(str(PureWindowsPath(r'R:\media', name, 'movie.mp4'))))@@ -396,7 +397,8 @@ class IngestPathTests(_ServiceCase):service = self.service()for parts in (('@Recently-Snapshot', 'movie.mp4'), ('.AppleDouble', 'movie.mp4'),('.zfs', 'snapshot', 'movie.mp4'), ('$Recycle.Bin', 'movie.mp4'),- ('._movie.mp4',), ('desktop.ini',)):+ ('._movie.mp4',), ('desktop.ini',), ('@eaDir',), ('.Trash-1000',),+ ('.snapshots',), ('.TemporaryItems',)):path = self.ledger('local', *parts)with self.subTest(parts=parts), mock.patch.object(service.queue, 'submit') as submit, \mock.patch.object(service, '_ingest') as ingest:diff --git a/tests/test_scan.py b/tests/test_scan.pyindex 3394207e1..0f133d944 100644--- a/tests/test_scan.py+++ b/tests/test_scan.py@@ -591,7 +591,8 @@ class NasDirectoryTests(unittest.TestCase):root = Path(temporary).resolve()for name in ("#recycle", "#RECYCLE", "@eaDir", "@snapshot", "@Recycle","@Recently-Snapshot", ".@__thumb", ".@__qini", ".@upload_cache", ".streams",- ".zfs", ".snapshot", "~snapshot", ".recycle", ".AppleDouble", ".AppleDB",+ ".zfs", ".snapshot", ".snapshots", "~snapshot", ".recycle", ".Trash", ".Trash-1000",+ ".TemporaryItems", ".AppleDouble", ".AppleDB",".AppleDesktop", "Network Trash Folder", ".Spotlight-V100", ".Trashes",".fseventsd", "__MACOSX", "$Recycle.Bin", "System Volume Information",".wfm_upload123", "normal", "@creator", ".hidden"):@@ -607,6 +608,31 @@ class NasDirectoryTests(unittest.TestCase):class SystemMetadataTests(_ScanCase):+ def test_a_system_directory_scan_root_is_skipped_before_mapping_or_listing(self):+ options = dict(declared_roots={'local': (r'R:\media\.snapshots',)},+ mounts={'local': (self.media,)}, windows=False)+ progress = []+ with patch.object(scan, 'walk_root_for') as mapped, patch.object(scan, '_walk') as walked:+ result, output = self._scan(r'R:\media\.snapshots', **options,+ progress=lambda files, total: progress.append((files, total)))+ self.assertEqual((result.files, result.total_bytes, result.gone), (0, 0, 0))+ self.assertEqual(progress, [(0, 0)])+ self.assertIn('0 文件', output)+ mapped.assert_not_called()+ walked.assert_not_called()+ self.assertEqual(self._paths(), {})+ with self.assertRaises(scan.ScanTargetError):+ self._scan(r'R:\media\.snapshots', declared_roots={})++ def test_system_walk_roots_and_resumed_directories_are_not_listed(self):+ system = self.media / '.Trash-1000'+ system.mkdir()+ (system / 'movie.mp4').write_bytes(b'media')+ with patch.object(scan.os, 'scandir') as listed:+ self.assertEqual(list(scan._walk(system)), [])+ self.assertEqual(list(scan._walk(self.media, [str(system)])), [])+ listed.assert_not_called()+def test_full_scan_and_direct_ingest_share_metadata_file_rules(self):names = ('desktop.ini', 'Thumbs.db', '.DS_Store', '._movie.mp4', '.SYNOPPSDB123','.wfm_upload123.mp4', 'movie.mp4@SynoEAStream')diff --git a/tests/test_smb_media.py b/tests/test_smb_media.pyindex 4b06115b8..043d52e84 100644--- a/tests/test_smb_media.py+++ b/tests/test_smb_media.py@@ -46,6 +46,18 @@ class _Resource:class SmbMediaTests(unittest.TestCase):+ def test_system_smb_walk_roots_and_resumed_directories_are_not_listed(self):+ root = smb_media.SmbResource('fixture', 'R:/Media', 'smb://files.invalid/media', (),+ SmbConnection('files.invalid', 'media'))+ with patch.object(smb_media.SmbResource, '_transport') as connect:+ self.assertEqual(list(scan._walk(root / '.snapshots')), [])+ connect.assert_not_called()+ transport = SimpleNamespace(close=lambda: None)+ with patch.object(smb_media.SmbResource, '_transport', return_value=transport), \+ patch.object(smb_media.SmbResource, 'scandir') as listed:+ self.assertEqual(list(scan._walk(root, [str(root / '.snapshots')])), [])+ listed.assert_not_called()+def tearDown(self):smb_media.close_bridge()diff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.pyindex 77142f7a5..664988c91 100644--- a/tests/test_upgrade_repairs.py+++ b/tests/test_upgrade_repairs.py@@ -195,6 +195,27 @@ class UpgradeRepairTests(unittest.TestCase):with self.database.read_connection() as connection:self.assertIsNone(connection.execute('SELECT disposal FROM asset WHERE id=1').fetchone()[0])+ def test_prior_restore_uses_windows_path_identity_without_exempting_other_paths(self):+ from peach import system_file_repair+ with self.database.write_transaction() as connection:+ for key in (1, 2):+ connection.execute("INSERT INTO asset(id,location,path,name,medium) "+ "VALUES(?,'local',?,'metadata','other')",+ (key, fr'R:\media\@eaDir\old{key}@SynoEAStream'))+ with patch.object(system_file_repair, 'RULE', system_file_repair.PRIOR_RULE):+ self.service.prepare()+ with self.database.write_transaction() as connection:+ connection.execute("UPDATE asset SET disposal=NULL,disposal_reason=NULL,path=? WHERE id=1",+ ('r:/MEDIA/@EADIR/OLD1@SYNOEASTREAM',))+ connection.execute("UPDATE asset SET disposal=NULL,disposal_reason=NULL,path=? WHERE id=2",+ (r'R:\media\@eaDir\different@SynoEAStream',))+ self.service.prepare()+ with self.database.read_connection() as connection:+ self.assertEqual([tuple(row) for row in connection.execute('SELECT id,disposal FROM asset ORDER BY id')],+ [(1, None), (2, 'trash')])+ receipt = upgrade_repairs._read(self.service.receipt)[system_file_repair.RULE]+ self.assertEqual([row['id'] for row in receipt['isolated']], [2])+def test_expanded_system_rules_preserve_prior_restores_and_personal_records(self):from peach import system_file_repairwith self.database.write_transaction() as connection: