Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(discovery): 补齐系统目录边界与还原路径保护

longmeidao9d1b9e353f48
diff --git a/docs/REUSE.md b/docs/REUSE.md
index 86a71e227..4f6422f87 100644
--- a/docs/REUSE.md
+++ b/docs/REUSE.md
@@ -1193,5 +1193,7 @@ Python 源文件纳入 `src/peach/library_sharing.py` 与 `src/peach/library_pro
- TrueNAS 的 `.zfs/snapshot` 见 [CORE 快照恢复说明](https://www.truenas.com/docs/core/13.3/coretutorials/printview/);NetApp 的 [`.snapshot`](https://docs.netapp.com/us-en/ontap/data-protection/snapshot-copies-work-concept.html) 与 [`~snapshot`](https://docs.netapp.com/us-en/ontap/smb-admin/access-snapshot-directories-clients-smb2-concept.html) 使用目录段精确匹配。Samba 仅采用 [默认 `.recycle`](https://www.samba.org/samba/docs/current/man-html/vfs_recycle.8.html),自定义回收站名称不能从名字推断。
- AFP 的 `.AppleDouble`、`.AppleDesktop`、`.AppleDB`、`Network Trash Folder`、`Icon\r`、`.VolumeIcon.icns` 见 [Netatalk 专用文件与目录](https://netatalk.io/docs/Special-Files-and-Folders)。macOS 的 `.Spotlight-V100`、`.Trashes`、`.fseventsd` 可见 [Apple Remote Desktop 指南](https://images.apple.com/server/docs/ARD_3_Admin_Guide_v3.3.pdf),`__MACOSX` 中的 AppleDouble 见 [Apple DTS 取证示例](https://developer.apple.com/forums/thread/772398)。`.DS_Store` 见 [Apple SMB 说明](https://support.apple.com/102064),`.localized` 沿用已有文件名规则。
- Windows 的 [`System Volume Information`](https://learn.microsoft.com/en-us/windows-hardware/drivers/ddi/fltkernel/nf-fltkernel-fltcreatesystemvolumeinformationfolder)、[`$Recycle.Bin`](https://learn.microsoft.com/en-us/troubleshoot/windows-client/shell-experience/recycle-bin-corrupted)、[`desktop.ini`](https://learn.microsoft.com/en-us/windows/win32/shell/how-to-customize-folders-with-desktop-ini) 和 [`Thumbs.db`](https://learn.microsoft.com/en-us/windows/client-management/mdm/policy-csp-admx-thumbnails) 分别按目录或文件名称匹配。
+- Linux 挂载点的 `.Trash` 和 `.Trash-<UID>` 见 [FreeDesktop Trash Specification](https://specifications.freedesktop.org/trash/1.0/),UID 只匹配完整 ASCII 十进制数字;`.Trash-user` 和 `.Trash-1000-old` 仍可登记。Snapper 管理的 Btrfs 快照目录 `.snapshots` 见 [openSUSE 说明](https://doc.opensuse.org/documentation/tumbleweed/snapper/);macOS `.TemporaryItems` 见 [Apple 文件系统模板](https://github.com/apple-oss-distributions/files/blob/main/hierarchy.osx)。这些名称按完整目录段匹配,目录本身作为扫描根、恢复断点或推送路径也跳过,不进入内部文件。
+- TerraMaster 支持团队确认的 [`#recycle`](https://forum.terra-master.com/en/viewtopic.php?t=2443) 已由同名规则覆盖。[ASUSTOR](https://www.asustor.com/en/online/online_help?id=11)、[fnOS](https://help.fnnas.com/articles/v1/file/team) 和 [UGREEN](https://support.ugnas.com/detail/article/en-US/159) 的已查官方说明介绍回收站功能,未据此推导物理目录名称,也不将普通 `Recycle Bin`、`Trash` 或中文文件夹名加入过滤。
-不排除任意 `@` 目录或任意点开头文件;`@creator/movie.mp4`、`.hidden.mp4`、`@eaDir-old` 和普通 `Snapshots` 仍可登记。旧数据沿用 `system_file_repair` 可还原隔离,仅处理无番号、无收藏和个人记录的 `other` 行,保留文件与关联行;上一版回执已经隔离过的相同资产与路径不再处理,以保留后来的手动还原。旧视频、图片和音频不自动隔离;过滤生效不代表这些历史资产已复核。
+不排除任意 `@` 目录或任意点开头文件;`@creator/movie.mp4`、`.hidden.mp4`、`@eaDir-old` 和普通 `Snapshots` 仍可登记。旧数据沿用 `system_file_repair` 可还原隔离,仅处理无番号、无收藏和个人记录的 `other` 行,保留文件与关联行;上一版回执已经隔离过的相同资产与 Windows 路径不再处理,以保留后来的手动还原。路径按 `PureWindowsPath` 比较,大小写和斜杠变化不影响保护,同一 ID 改为另一路径仍按规则检查。旧视频、图片和音频不自动隔离;过滤生效不代表这些历史资产已复核。
diff --git a/src/peach/discovery_rules.py b/src/peach/discovery_rules.py
index 72428f6cf..940b9246c 100644
--- a/src/peach/discovery_rules.py
+++ b/src/peach/discovery_rules.py
@@ -1,14 +1,15 @@
"""全量扫描、推送与下载登记共用的系统元数据判据。"""
from pathlib import PureWindowsPath
+import re
# 厂商保留名称的来源与边界见 docs/REUSE.md「系统元数据发现过滤」。
_DIRECTORIES = frozenset({
'@eadir', '@tmp', '@sharebin', '#recycle', '#snapshot', '@snapshot',
'@recycle', '@recently-snapshot', '.@__thumb', '.@__qini', '.@upload_cache', '.streams',
- '.zfs', '.snapshot', '~snapshot', '.recycle',
+ '.zfs', '.snapshot', '.snapshots', '~snapshot', '.recycle', '.trash',
'.appledouble', '.appledesktop', '.appledb', 'network trash folder',
- '.spotlight-v100', '.trashes', '.fseventsd', '__macosx',
+ '.spotlight-v100', '.trashes', '.temporaryitems', '.fseventsd', '__macosx',
'$recycle.bin', 'system volume information',
})
_FILES = frozenset({
@@ -18,7 +19,8 @@ _FILES = frozenset({
def system_directory(name: str) -> bool:
lowered = name.casefold()
- return lowered in _DIRECTORIES or lowered.startswith('.wfm_upload')
+ return (lowered in _DIRECTORIES or lowered.startswith('.wfm_upload')
+ or re.fullmatch(r'\.trash-[0-9]+', lowered) is not None)
def system_file(name: str) -> bool:
@@ -29,5 +31,5 @@ def system_file(name: str) -> bool:
def system_path(path: str) -> bool:
parts = PureWindowsPath(path).parts
- return any(system_directory(part) for part in parts[:-1]) or (
+ return any(system_directory(part) for part in parts) or (
bool(parts) and system_file(parts[-1]))
diff --git a/src/peach/push_discovery.py b/src/peach/push_discovery.py
index d4e5f7823..d6fefa374 100644
--- a/src/peach/push_discovery.py
+++ b/src/peach/push_discovery.py
@@ -44,7 +44,7 @@ from typing import Callable
from .fsutil import atomic_write_text
from .repository import in_background
-from .discovery_rules import system_directory, system_path
+from .discovery_rules import system_path
LOGGER = logging.getLogger(__name__)
@@ -695,7 +695,7 @@ class PushDiscoveryService:
return self.queue.submit(location, path, time.monotonic())
def submit_move(self, location, old, new, *, directory=False):
- if ignored(new) or (directory and system_directory(PureWindowsPath(new).name)):
+ if ignored(new):
return
if directory:
from .media_folders import path_predicate
diff --git a/src/peach/scan.py b/src/peach/scan.py
index bfdec5ad0..e6c697ea0 100644
--- a/src/peach/scan.py
+++ b/src/peach/scan.py
@@ -482,6 +482,8 @@ def _walk(top: Path, pending: list[str] | None = None, *, onerror=lambda error:
列表自带这两项,不必再对每个文件发一次 `os.stat`——网盘挂载上那一次就是一趟往返,
几万个文件就是几万趟。读不了的目录跳过;符号链接指向的目录列出但不进入。
"""
+ if system_path(str(top)):
+ return
from .smb_media import SmbResource
if isinstance(top, SmbResource):
yield from _walk_smb(top, pending, onerror)
@@ -490,6 +492,8 @@ def _walk(top: Path, pending: list[str] | None = None, *, onerror=lambda error:
while stack:
checkpoint('scan')
directory = stack.pop()
+ if system_path(directory):
+ continue
try:
if read_directory is not None:
entries = read_directory(directory)
@@ -540,6 +544,8 @@ def _walk_smb(top, pending, onerror):
while stack:
checkpoint('scan')
directory = stack.pop()
+ if system_path(directory):
+ continue
resource = top.joinpath(*_relative_directory(directory, top))
try:
try:
@@ -585,6 +591,12 @@ class _ScanProgress:
def scan_location(db_path, location, root, **kwargs) -> ScanResult:
"""命令行按来源与根目录保存断点,组合任务可提供自己的持久检查点。"""
+ if system_path(root):
+ check_scan_target(location, root, declared_roots=kwargs['declared_roots'])
+ result = ScanResult(location, root, 0, 0, 0, 0)
+ kwargs.get('progress', lambda files, total: None)(0, 0)
+ kwargs.get('report', print)(result.summary())
+ return result
if 'save_checkpoint' in kwargs or 'resume_state' in kwargs:
return _scan_location(db_path, location, root, **kwargs)
from .update_tasks import state_for
diff --git a/src/peach/system_file_repair.py b/src/peach/system_file_repair.py
index db6668eb0..de815e994 100644
--- a/src/peach/system_file_repair.py
+++ b/src/peach/system_file_repair.py
@@ -1,5 +1,6 @@
"""将旧推送发现登记的系统元数据放入可还原回收站。"""
import json
+from pathlib import PureWindowsPath
from .discovery_rules import system_path
from .personal_records import record_holders
@@ -8,13 +9,13 @@ PRIOR_RULE = 'discovery-system-files-1'
RULE = 'discovery-system-files-2'
-def _previously_isolated(connection) -> set[tuple[int, str]]:
+def _previously_isolated(connection) -> set[tuple[int, PureWindowsPath]]:
"""旧回执中隔离过的同一路径不再处理,保留用户后来的还原决定。"""
receipts = connection.execute(
'SELECT json_extract(result_json,?) FROM ledger_upgrade_repair '
'WHERE json_type(result_json,?) IS NOT NULL',
(f'$."{PRIOR_RULE}"', f'$."{PRIOR_RULE}"'))
- return {(row['id'], row['path']) for receipt in receipts
+ return {(row['id'], PureWindowsPath(row['path'])) for receipt in receipts
for row in json.loads(receipt[0])['isolated']}
@@ -24,7 +25,7 @@ def collect(connection) -> list[dict]:
"SELECT id,path,disposal,disposal_reason,feedback_at FROM asset "
"WHERE disposal IS NULL AND medium='other' AND coalesce(code,'')='' "
"AND coalesce(feedback,'')<>'like'")
- if system_path(row['path']) and (row['id'], row['path']) not in previous]
+ if system_path(row['path']) and (row['id'], PureWindowsPath(row['path'])) not in previous]
protected = record_holders(connection, [row['id'] for row in rows])
return [row for row in rows if row['id'] not in protected and not connection.execute(
"SELECT 1 FROM review_decision WHERE category='junk_file' AND item_key=? AND status='rejected'",
diff --git a/tests/test_push_discovery.py b/tests/test_push_discovery.py
index fde60660d..147fcd9ed 100644
--- a/tests/test_push_discovery.py
+++ b/tests/test_push_discovery.py
@@ -211,7 +211,8 @@ class IgnoreAndSourceTests(unittest.TestCase):
with self.subTest(name=repr(name)):
self.assertTrue(push.ignored(str(PureWindowsPath(r'R:\media', name))))
for name in ('@creator', 'normal', '@eaDir-old', '#recycle-old', '.snapshot-old',
- 'Snapshots', 'Recycle Bin', 'Temporary Items'):
+ 'Snapshots', 'Recycle Bin', 'Temporary Items', '.Trash-', '.Trash-1000-old',
+ '.Trash-user', '.Trash-١٠٠٠', '.snapshots-old', '.TemporaryItems-old'):
with self.subTest(directory=name):
self.assertFalse(system_directory(name))
self.assertFalse(push.ignored(str(PureWindowsPath(r'R:\media', name, 'movie.mp4'))))
@@ -396,7 +397,8 @@ class IngestPathTests(_ServiceCase):
service = self.service()
for parts in (('@Recently-Snapshot', 'movie.mp4'), ('.AppleDouble', 'movie.mp4'),
('.zfs', 'snapshot', 'movie.mp4'), ('$Recycle.Bin', 'movie.mp4'),
- ('._movie.mp4',), ('desktop.ini',)):
+ ('._movie.mp4',), ('desktop.ini',), ('@eaDir',), ('.Trash-1000',),
+ ('.snapshots',), ('.TemporaryItems',)):
path = self.ledger('local', *parts)
with self.subTest(parts=parts), mock.patch.object(service.queue, 'submit') as submit, \
mock.patch.object(service, '_ingest') as ingest:
diff --git a/tests/test_scan.py b/tests/test_scan.py
index 3394207e1..0f133d944 100644
--- a/tests/test_scan.py
+++ b/tests/test_scan.py
@@ -591,7 +591,8 @@ class NasDirectoryTests(unittest.TestCase):
root = Path(temporary).resolve()
for name in ("#recycle", "#RECYCLE", "@eaDir", "@snapshot", "@Recycle",
"@Recently-Snapshot", ".@__thumb", ".@__qini", ".@upload_cache", ".streams",
- ".zfs", ".snapshot", "~snapshot", ".recycle", ".AppleDouble", ".AppleDB",
+ ".zfs", ".snapshot", ".snapshots", "~snapshot", ".recycle", ".Trash", ".Trash-1000",
+ ".TemporaryItems", ".AppleDouble", ".AppleDB",
".AppleDesktop", "Network Trash Folder", ".Spotlight-V100", ".Trashes",
".fseventsd", "__MACOSX", "$Recycle.Bin", "System Volume Information",
".wfm_upload123", "normal", "@creator", ".hidden"):
@@ -607,6 +608,31 @@ class NasDirectoryTests(unittest.TestCase):
class SystemMetadataTests(_ScanCase):
+ def test_a_system_directory_scan_root_is_skipped_before_mapping_or_listing(self):
+ options = dict(declared_roots={'local': (r'R:\media\.snapshots',)},
+ mounts={'local': (self.media,)}, windows=False)
+ progress = []
+ with patch.object(scan, 'walk_root_for') as mapped, patch.object(scan, '_walk') as walked:
+ result, output = self._scan(r'R:\media\.snapshots', **options,
+ progress=lambda files, total: progress.append((files, total)))
+ self.assertEqual((result.files, result.total_bytes, result.gone), (0, 0, 0))
+ self.assertEqual(progress, [(0, 0)])
+ self.assertIn('0 文件', output)
+ mapped.assert_not_called()
+ walked.assert_not_called()
+ self.assertEqual(self._paths(), {})
+ with self.assertRaises(scan.ScanTargetError):
+ self._scan(r'R:\media\.snapshots', declared_roots={})
+
+ def test_system_walk_roots_and_resumed_directories_are_not_listed(self):
+ system = self.media / '.Trash-1000'
+ system.mkdir()
+ (system / 'movie.mp4').write_bytes(b'media')
+ with patch.object(scan.os, 'scandir') as listed:
+ self.assertEqual(list(scan._walk(system)), [])
+ self.assertEqual(list(scan._walk(self.media, [str(system)])), [])
+ listed.assert_not_called()
+
def test_full_scan_and_direct_ingest_share_metadata_file_rules(self):
names = ('desktop.ini', 'Thumbs.db', '.DS_Store', '._movie.mp4', '.SYNOPPSDB123',
'.wfm_upload123.mp4', 'movie.mp4@SynoEAStream')
diff --git a/tests/test_smb_media.py b/tests/test_smb_media.py
index 4b06115b8..043d52e84 100644
--- a/tests/test_smb_media.py
+++ b/tests/test_smb_media.py
@@ -46,6 +46,18 @@ class _Resource:
class SmbMediaTests(unittest.TestCase):
+ def test_system_smb_walk_roots_and_resumed_directories_are_not_listed(self):
+ root = smb_media.SmbResource('fixture', 'R:/Media', 'smb://files.invalid/media', (),
+ SmbConnection('files.invalid', 'media'))
+ with patch.object(smb_media.SmbResource, '_transport') as connect:
+ self.assertEqual(list(scan._walk(root / '.snapshots')), [])
+ connect.assert_not_called()
+ transport = SimpleNamespace(close=lambda: None)
+ with patch.object(smb_media.SmbResource, '_transport', return_value=transport), \
+ patch.object(smb_media.SmbResource, 'scandir') as listed:
+ self.assertEqual(list(scan._walk(root, [str(root / '.snapshots')])), [])
+ listed.assert_not_called()
+
def tearDown(self):
smb_media.close_bridge()
diff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.py
index 77142f7a5..664988c91 100644
--- a/tests/test_upgrade_repairs.py
+++ b/tests/test_upgrade_repairs.py
@@ -195,6 +195,27 @@ class UpgradeRepairTests(unittest.TestCase):
with self.database.read_connection() as connection:
self.assertIsNone(connection.execute('SELECT disposal FROM asset WHERE id=1').fetchone()[0])
+ def test_prior_restore_uses_windows_path_identity_without_exempting_other_paths(self):
+ from peach import system_file_repair
+ with self.database.write_transaction() as connection:
+ for key in (1, 2):
+ connection.execute("INSERT INTO asset(id,location,path,name,medium) "
+ "VALUES(?,'local',?,'metadata','other')",
+ (key, fr'R:\media\@eaDir\old{key}@SynoEAStream'))
+ with patch.object(system_file_repair, 'RULE', system_file_repair.PRIOR_RULE):
+ self.service.prepare()
+ with self.database.write_transaction() as connection:
+ connection.execute("UPDATE asset SET disposal=NULL,disposal_reason=NULL,path=? WHERE id=1",
+ ('r:/MEDIA/@EADIR/OLD1@SYNOEASTREAM',))
+ connection.execute("UPDATE asset SET disposal=NULL,disposal_reason=NULL,path=? WHERE id=2",
+ (r'R:\media\@eaDir\different@SynoEAStream',))
+ self.service.prepare()
+ with self.database.read_connection() as connection:
+ self.assertEqual([tuple(row) for row in connection.execute('SELECT id,disposal FROM asset ORDER BY id')],
+ [(1, None), (2, 'trash')])
+ receipt = upgrade_repairs._read(self.service.receipt)[system_file_repair.RULE]
+ self.assertEqual([row['id'] for row in receipt['isolated']], [2])
+
def test_expanded_system_rules_preserve_prior_restores_and_personal_records(self):
from peach import system_file_repair
with self.database.write_transaction() as connection: