Peach: Local-first video metadata scraper and media manager.
fix(discovery): 统一系统路径过滤并隔离旧群晖元数据
longmeidao8a8eda729b34
diff --git a/src/peach/discovery_rules.py b/src/peach/discovery_rules.pynew file mode 100644index 000000000..3f31ad943--- /dev/null+++ b/src/peach/discovery_rules.py@@ -0,0 +1,12 @@+"""全量扫描和推送发现共用的系统目录判据。"""+from pathlib import PureWindowsPath+++def system_directory(name: str) -> bool:+ return name.startswith('@') or name.casefold() == '#recycle'+++def system_path(path: str) -> bool:+ parts = PureWindowsPath(path).parts+ return any(system_directory(part) for part in parts[:-1]) or (+ bool(parts) and parts[-1].casefold().endswith('@synoeastream'))diff --git a/src/peach/push_discovery.py b/src/peach/push_discovery.pyindex cf4d629a3..784e78414 100644--- a/src/peach/push_discovery.py+++ b/src/peach/push_discovery.py@@ -44,6 +44,7 @@ from typing import Callablefrom .fsutil import atomic_write_textfrom .repository import in_background+from .discovery_rules import system_directory, system_pathLOGGER = logging.getLogger(__name__)@@ -92,9 +93,11 @@ class CloudPathError(ValueError):"""云端路径映射不出账本路径。消息可以直接给人看。"""-def ignored(name: str) -> bool:- """这个文件名要不要直接丢掉。"""- lowered = name.casefold()+def ignored(path: str) -> bool:+ """系统目录、系统文件与半成品不进入登记队列。"""+ if system_path(path):+ return True+ lowered = PureWindowsPath(path).name.casefold()if lowered in IGNORED_NAMES or lowered.startswith("."):return Truereturn os.path.splitext(lowered)[1] in IGNORED_SUFFIXES@@ -694,11 +697,13 @@ class PushDiscoveryService:LOGGER.exception("推送发现探测时长失败:%s", path)def submit(self, location: str, path: str) -> bool:- if ignored(PureWindowsPath(path).name):+ if ignored(path):return Falsereturn self.queue.submit(location, path, time.monotonic())def submit_move(self, location, old, new, *, directory=False):+ if ignored(new) or (directory and system_directory(PureWindowsPath(new).name)):+ returnif directory:from .media_folders import path_predicatefrom contextlib import closing@@ -709,8 +714,6 @@ class PushDiscoveryService:tail = PureWindowsPath(path).relative_to(PureWindowsPath(old))self.submit_move(location, path, str(PureWindowsPath(new) / tail))return- if ignored(PureWindowsPath(new).name):- returnself.queue.submit(location, new, time.monotonic(), move_from=old)def ingest_now(self, location: str, path: str) -> bool:@@ -719,7 +722,7 @@ class PushDiscoveryService:云下载对账时用:远端已经完成、文件就在挂载目录里,推送发现那一侧却没送来通知(没开、没配会员,或通知丢了)。文件不在返回假。"""- if ignored(PureWindowsPath(path).name):+ if ignored(path):return False# 云下载整理完成时另有一条通知,这里落地登记不再弹。return self._ingest(location, path, notify=False)diff --git a/src/peach/scan.py b/src/peach/scan.pyindex eb5c332b6..565d0829a 100644--- a/src/peach/scan.py+++ b/src/peach/scan.py@@ -49,6 +49,7 @@ from . import folder_rules, junk, record_rehome, subtitles, wantsfrom .platform import is_windows_path, is_unmapped, resolve_location, resolve_rootfrom .processing_control import checkpointfrom .fsutil import atomic_write_text+from .discovery_rules import system_directory, system_pathfrom .catalog_rules import AUDIO_EXTENSIONS, VIDEO_EXTENSIONS, is_local_extraVIDEO = VIDEO_EXTENSIONS@@ -378,7 +379,8 @@ def ingest_path(except OSError:return IngestResult(location, str(ledger_path), False)library_roots = tuple(chain.from_iterable(declared_roots.values()))- if _sidecar_on_disk(name, directory, ledger_path=str(ledger_path), library_roots=library_roots):+ if system_path(str(ledger_path)) or _sidecar_on_disk(+ name, directory, ledger_path=str(ledger_path), library_roots=library_roots):return IngestResult(location, str(ledger_path), True, stat.st_size, sidecar=True)now = time.strftime("%Y-%m-%d %H:%M:%S")mtime = time.strftime("%Y-%m-%d", time.localtime(stat.st_mtime))@@ -505,7 +507,7 @@ def _walk(top: Path, pending: list[str] | None = None, *, onerror=lambda error:is_dir = Falseif is_dir:try:- if not entry.is_symlink() and entry.name.casefold() != "#recycle" and not entry.name.startswith("@"):+ if not entry.is_symlink() and not system_directory(entry.name):subdirectories.append(entry.path)except OSError:pass@@ -556,7 +558,7 @@ def _walk_smb(top, pending, onerror):continuepath = str(resource / entry.name)if entry.is_dir:- if not entry.name.startswith('@') and entry.name.casefold() != '#recycle':+ if not system_directory(entry.name):children.append(path)else:stat = SimpleNamespace(st_size=entry.size, st_mtime=entry.mtime, st_mtime_ns=entry.mtime_ns)@@ -747,8 +749,8 @@ def _scan_location(if cursor is not None and name <= cursor:continueledger_path = str(ledger_root.joinpath(*relative, name))- if is_sidecar(name, videos, artwork_dir=artwork_dir,- path=ledger_path, library_roots=library_roots):+ if system_path(ledger_path) or is_sidecar(name, videos, artwork_dir=artwork_dir,+ path=ledger_path, library_roots=library_roots):skipped += 1continueif new_only and ledger_path.casefold() in registered:diff --git a/src/peach/system_file_repair.py b/src/peach/system_file_repair.pynew file mode 100644index 000000000..10a75996e--- /dev/null+++ b/src/peach/system_file_repair.py@@ -0,0 +1,29 @@+"""将旧推送发现登记的系统元数据放入可还原回收站。"""+from .discovery_rules import system_path+from .personal_records import record_holders++RULE = 'discovery-system-files-1'+++def collect(connection) -> list[dict]:+ rows = [dict(row) for row in connection.execute(+ "SELECT id,path,disposal,disposal_reason,feedback_at FROM asset "+ "WHERE disposal IS NULL AND medium='other' AND coalesce(code,'')='' "+ "AND coalesce(feedback,'')<>'like'") if system_path(row['path'])]+ protected = record_holders(connection, [row['id'] for row in rows])+ return [row for row in rows if row['id'] not in protected and not connection.execute(+ "SELECT 1 FROM review_decision WHERE category='junk_file' AND item_key=? AND status='rejected'",+ (str(row['id']),)).fetchone()]+++def apply(connection, rows, stamp: float) -> list[dict]:+ current = {row['id']: row for row in collect(connection)}+ changed = []+ for row in rows:+ if current.get(row['id']) != row:+ continue+ connection.execute(+ "UPDATE asset SET disposal='trash',disposal_reason='junk',feedback_at=? WHERE id=?",+ (stamp, row['id']))+ changed.append(row)+ return changeddiff --git a/src/peach/upgrade_repairs.py b/src/peach/upgrade_repairs.pyindex 0a519b0b3..d00d669a2 100644--- a/src/peach/upgrade_repairs.py+++ b/src/peach/upgrade_repairs.py@@ -315,6 +315,7 @@ class UpgradeRepairs:if applied is not None:self._restore_receipt(json.loads(applied[0]))self._withdraw_stale_code_evidence()+ self._isolate_system_files()returnself._restore_receipt(None)sources = Path(contract.follow_sources_root)@@ -356,6 +357,37 @@ class UpgradeRepairs:(REVISION, str(time.time()), json.dumps(receipt, ensure_ascii=False)))atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))self._withdraw_stale_code_evidence()+ self._isolate_system_files()++ def _isolate_system_files(self) -> None:+ """旧系统元数据按路径规则隔离;回执保留原状态,媒体与关联行不动。"""+ from . import system_file_repair+ with self.contract.database.read_connection() as connection:+ stored = connection.execute('SELECT result_json FROM ledger_upgrade_repair WHERE id=?',+ (REVISION,)).fetchone()+ receipt = json.loads(stored[0]) if stored else None+ if receipt is None or system_file_repair.RULE in receipt:+ return+ earlier = connection.execute(+ 'SELECT result_json FROM ledger_upgrade_repair WHERE json_type(result_json,?) IS NOT NULL '+ 'ORDER BY applied_at DESC LIMIT 1', (f'$."{system_file_repair.RULE}"',)).fetchone()+ if earlier:+ receipt[system_file_repair.RULE] = json.loads(earlier[0])[system_file_repair.RULE]+ else:+ rows = system_file_repair.collect(connection)+ if earlier:+ self._save_receipt(receipt)+ return+ backup = None+ if rows:+ backup = self.contract.db_path.with_name(f'ledger.pre-upgrade-{time.time_ns()}.db')+ sqlite_backup(self.contract.db_path, backup)+ with self.contract.database.write_transaction() as connection:+ changed = system_file_repair.apply(connection, rows, time.time())+ receipt[system_file_repair.RULE] = {'isolated': changed, 'backup': str(backup) if backup else None}+ connection.execute('UPDATE ledger_upgrade_repair SET result_json=? WHERE id=?',+ (json.dumps(receipt, ensure_ascii=False), REVISION))+ atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))def _withdraw_stale_code_evidence(self) -> None:"""撤掉旧番号下取到的候选与快照(`stale_code_evidence`),被撤的影片按现番号重采。diff --git a/tests/test_push_discovery.py b/tests/test_push_discovery.pyindex 9dfea2b5d..913ea15da 100644--- a/tests/test_push_discovery.py+++ b/tests/test_push_discovery.py@@ -196,6 +196,15 @@ class IgnoreAndSourceTests(unittest.TestCase):self.assertTrue(push.ignored(name))self.assertFalse(push.ignored("正片.mp4"))+ def test_system_directories_are_ignored_before_names_are_encoded(self):+ for path in (r'R:\media\@eaDir\video.mp4@SynoEAStream',+ '/media/@snapshot/nested/video.mp4', r'R:\media\#RECYCLE\video.mp4',+ 'R:\\media\\@eaDir\\broken\udcff.mp4@SynoEAStream',+ r'R:\media\video.mp4@SynoEAStream'):+ with self.subTest(path=ascii(path)):+ self.assertTrue(push.ignored(path))+ self.assertFalse(push.ignored(r'R:\media\scene@home\video.mp4'))+def test_only_loopback_and_private_addresses_are_accepted(self):# 局域网那一档用 RFC 5737 的文档网段举例:`ipaddress` 把它判成 private,# 正是 `allowed_source` 看的那一位,写谁家的真实网段都不必要(`test_repo_hygiene`)。@@ -349,6 +358,27 @@ class _ServiceCase(unittest.TestCase):class IngestPathTests(_ServiceCase):+ def test_system_files_never_enter_any_push_registration_path(self):+ service = self.service()+ path = self.ledger('local', '@eaDir', 'broken\udcff.mp4@SynoEAStream')+ with mock.patch.object(service.queue, 'submit') as submit, \+ mock.patch.object(service, '_ingest') as ingest:+ self.assertFalse(service.submit('local', path))+ service.submit_move('local', self.ledger('local', 'old.mp4'), path)+ service.submit_move('local', path, self.ledger('local', '@eaDir'), directory=True)+ self.assertFalse(service.ingest_now('local', path))+ submit.assert_not_called()+ ingest.assert_not_called()++ def test_direct_ingest_does_not_register_system_directory_contents(self):+ file = self.media / '@eaDir/video.mp4@SynoEAStream'+ file.parent.mkdir()+ file.write_bytes(b'metadata')+ result = scan.ingest_path(self.db, 'local', self.ledger('local', '@eaDir', file.name),+ declared_roots=self.declared, mounts=self.mounts)+ self.assertTrue(result.sidecar)+ self.assertEqual(self.rows(), {})+def test_move_event_keeps_upgrade_existing_asset_without_an_identity_cache(self):from contextlib import closingold = self.media / 'old.mp4'diff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.pyindex 779c6396e..b933e27f3 100644--- a/tests/test_upgrade_repairs.py+++ b/tests/test_upgrade_repairs.py@@ -15,6 +15,39 @@ from support.ledger import fresh_ledgerclass UpgradeRepairTests(unittest.TestCase):+ def test_old_system_metadata_is_isolated_once_without_deleting_files_or_records(self):+ from peach import system_file_repair+ file = self.root / '@eaDir/video.mp4@SynoEAStream'+ file.parent.mkdir()+ file.write_bytes(b'metadata')+ rows = [(1, str(file), None), (2, r'R:\media\@eaDir\kept@SynoEAStream', 5),+ (3, r'R:\media\normal.txt', None)]+ with self.database.write_transaction() as connection:+ for key, path, rating in rows:+ connection.execute("INSERT INTO asset(id,location,path,name,medium,rating) "+ "VALUES(?,'local',?,'metadata','other',?)", (key, path, rating))+ connection.execute("INSERT INTO asset_tag(asset_id,tag,source) VALUES(1,'retained','test')")+ self.service.prepare()+ self.service.prepare()+ with self.database.read_connection() as connection:+ self.assertEqual([tuple(row) for row in connection.execute('SELECT id,disposal,rating FROM asset')],+ [(1, 'trash', None), (2, None, 5), (3, None, None)])+ self.assertEqual(connection.execute('SELECT count(*) FROM asset_tag WHERE asset_id=1').fetchone()[0], 1)+ self.assertEqual(file.read_bytes(), b'metadata')+ receipt = upgrade_repairs._read(self.service.receipt)[system_file_repair.RULE]+ self.assertEqual(receipt['isolated'], [dict(id=1, path=str(file), disposal=None,+ disposal_reason=None, feedback_at=None)])+ self.assertTrue(Path(receipt['backup']).is_file())+ with self.database.write_transaction() as connection:+ connection.execute('UPDATE asset SET disposal=NULL,disposal_reason=NULL WHERE id=1')+ self.service.prepare()+ with self.database.read_connection() as connection:+ self.assertIsNone(connection.execute('SELECT disposal FROM asset WHERE id=1').fetchone()[0])+ with patch.object(upgrade_repairs, 'REVISION', upgrade_repairs.REVISION + '-next'):+ self.service.prepare()+ with self.database.read_connection() as connection:+ self.assertIsNone(connection.execute('SELECT disposal FROM asset WHERE id=1').fetchone()[0])+def test_javdb_upgrade_requeues_legacy_evidence_and_keeps_manual_values(self):from peach import artifact_rules, upgrade_rulesself.asset(1, code='EXM-001', duration=1)