Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(discovery): 统一系统路径过滤并隔离旧群晖元数据

longmeidao8a8eda729b34
diff --git a/src/peach/discovery_rules.py b/src/peach/discovery_rules.py
new file mode 100644
index 000000000..3f31ad943
--- /dev/null
+++ b/src/peach/discovery_rules.py
@@ -0,0 +1,12 @@
+"""全量扫描和推送发现共用的系统目录判据。"""
+from pathlib import PureWindowsPath
+
+
+def system_directory(name: str) -> bool:
+ return name.startswith('@') or name.casefold() == '#recycle'
+
+
+def system_path(path: str) -> bool:
+ parts = PureWindowsPath(path).parts
+ return any(system_directory(part) for part in parts[:-1]) or (
+ bool(parts) and parts[-1].casefold().endswith('@synoeastream'))
diff --git a/src/peach/push_discovery.py b/src/peach/push_discovery.py
index cf4d629a3..784e78414 100644
--- a/src/peach/push_discovery.py
+++ b/src/peach/push_discovery.py
@@ -44,6 +44,7 @@ from typing import Callable
from .fsutil import atomic_write_text
from .repository import in_background
+from .discovery_rules import system_directory, system_path
LOGGER = logging.getLogger(__name__)
@@ -92,9 +93,11 @@ class CloudPathError(ValueError):
"""云端路径映射不出账本路径。消息可以直接给人看。"""
-def ignored(name: str) -> bool:
- """这个文件名要不要直接丢掉。"""
- lowered = name.casefold()
+def ignored(path: str) -> bool:
+ """系统目录、系统文件与半成品不进入登记队列。"""
+ if system_path(path):
+ return True
+ lowered = PureWindowsPath(path).name.casefold()
if lowered in IGNORED_NAMES or lowered.startswith("."):
return True
return os.path.splitext(lowered)[1] in IGNORED_SUFFIXES
@@ -694,11 +697,13 @@ class PushDiscoveryService:
LOGGER.exception("推送发现探测时长失败:%s", path)
def submit(self, location: str, path: str) -> bool:
- if ignored(PureWindowsPath(path).name):
+ if ignored(path):
return False
return self.queue.submit(location, path, time.monotonic())
def submit_move(self, location, old, new, *, directory=False):
+ if ignored(new) or (directory and system_directory(PureWindowsPath(new).name)):
+ return
if directory:
from .media_folders import path_predicate
from contextlib import closing
@@ -709,8 +714,6 @@ class PushDiscoveryService:
tail = PureWindowsPath(path).relative_to(PureWindowsPath(old))
self.submit_move(location, path, str(PureWindowsPath(new) / tail))
return
- if ignored(PureWindowsPath(new).name):
- return
self.queue.submit(location, new, time.monotonic(), move_from=old)
def ingest_now(self, location: str, path: str) -> bool:
@@ -719,7 +722,7 @@ class PushDiscoveryService:
云下载对账时用:远端已经完成、文件就在挂载目录里,推送发现那一侧却没送来通知
(没开、没配会员,或通知丢了)。文件不在返回假。
"""
- if ignored(PureWindowsPath(path).name):
+ if ignored(path):
return False
# 云下载整理完成时另有一条通知,这里落地登记不再弹。
return self._ingest(location, path, notify=False)
diff --git a/src/peach/scan.py b/src/peach/scan.py
index eb5c332b6..565d0829a 100644
--- a/src/peach/scan.py
+++ b/src/peach/scan.py
@@ -49,6 +49,7 @@ from . import folder_rules, junk, record_rehome, subtitles, wants
from .platform import is_windows_path, is_unmapped, resolve_location, resolve_root
from .processing_control import checkpoint
from .fsutil import atomic_write_text
+from .discovery_rules import system_directory, system_path
from .catalog_rules import AUDIO_EXTENSIONS, VIDEO_EXTENSIONS, is_local_extra
VIDEO = VIDEO_EXTENSIONS
@@ -378,7 +379,8 @@ def ingest_path(
except OSError:
return IngestResult(location, str(ledger_path), False)
library_roots = tuple(chain.from_iterable(declared_roots.values()))
- if _sidecar_on_disk(name, directory, ledger_path=str(ledger_path), library_roots=library_roots):
+ if system_path(str(ledger_path)) or _sidecar_on_disk(
+ name, directory, ledger_path=str(ledger_path), library_roots=library_roots):
return IngestResult(location, str(ledger_path), True, stat.st_size, sidecar=True)
now = time.strftime("%Y-%m-%d %H:%M:%S")
mtime = time.strftime("%Y-%m-%d", time.localtime(stat.st_mtime))
@@ -505,7 +507,7 @@ def _walk(top: Path, pending: list[str] | None = None, *, onerror=lambda error:
is_dir = False
if is_dir:
try:
- if not entry.is_symlink() and entry.name.casefold() != "#recycle" and not entry.name.startswith("@"):
+ if not entry.is_symlink() and not system_directory(entry.name):
subdirectories.append(entry.path)
except OSError:
pass
@@ -556,7 +558,7 @@ def _walk_smb(top, pending, onerror):
continue
path = str(resource / entry.name)
if entry.is_dir:
- if not entry.name.startswith('@') and entry.name.casefold() != '#recycle':
+ if not system_directory(entry.name):
children.append(path)
else:
stat = SimpleNamespace(st_size=entry.size, st_mtime=entry.mtime, st_mtime_ns=entry.mtime_ns)
@@ -747,8 +749,8 @@ def _scan_location(
if cursor is not None and name <= cursor:
continue
ledger_path = str(ledger_root.joinpath(*relative, name))
- if is_sidecar(name, videos, artwork_dir=artwork_dir,
- path=ledger_path, library_roots=library_roots):
+ if system_path(ledger_path) or is_sidecar(name, videos, artwork_dir=artwork_dir,
+ path=ledger_path, library_roots=library_roots):
skipped += 1
continue
if new_only and ledger_path.casefold() in registered:
diff --git a/src/peach/system_file_repair.py b/src/peach/system_file_repair.py
new file mode 100644
index 000000000..10a75996e
--- /dev/null
+++ b/src/peach/system_file_repair.py
@@ -0,0 +1,29 @@
+"""将旧推送发现登记的系统元数据放入可还原回收站。"""
+from .discovery_rules import system_path
+from .personal_records import record_holders
+
+RULE = 'discovery-system-files-1'
+
+
+def collect(connection) -> list[dict]:
+ rows = [dict(row) for row in connection.execute(
+ "SELECT id,path,disposal,disposal_reason,feedback_at FROM asset "
+ "WHERE disposal IS NULL AND medium='other' AND coalesce(code,'')='' "
+ "AND coalesce(feedback,'')<>'like'") if system_path(row['path'])]
+ protected = record_holders(connection, [row['id'] for row in rows])
+ return [row for row in rows if row['id'] not in protected and not connection.execute(
+ "SELECT 1 FROM review_decision WHERE category='junk_file' AND item_key=? AND status='rejected'",
+ (str(row['id']),)).fetchone()]
+
+
+def apply(connection, rows, stamp: float) -> list[dict]:
+ current = {row['id']: row for row in collect(connection)}
+ changed = []
+ for row in rows:
+ if current.get(row['id']) != row:
+ continue
+ connection.execute(
+ "UPDATE asset SET disposal='trash',disposal_reason='junk',feedback_at=? WHERE id=?",
+ (stamp, row['id']))
+ changed.append(row)
+ return changed
diff --git a/src/peach/upgrade_repairs.py b/src/peach/upgrade_repairs.py
index 0a519b0b3..d00d669a2 100644
--- a/src/peach/upgrade_repairs.py
+++ b/src/peach/upgrade_repairs.py
@@ -315,6 +315,7 @@ class UpgradeRepairs:
if applied is not None:
self._restore_receipt(json.loads(applied[0]))
self._withdraw_stale_code_evidence()
+ self._isolate_system_files()
return
self._restore_receipt(None)
sources = Path(contract.follow_sources_root)
@@ -356,6 +357,37 @@ class UpgradeRepairs:
(REVISION, str(time.time()), json.dumps(receipt, ensure_ascii=False)))
atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))
self._withdraw_stale_code_evidence()
+ self._isolate_system_files()
+
+ def _isolate_system_files(self) -> None:
+ """旧系统元数据按路径规则隔离;回执保留原状态,媒体与关联行不动。"""
+ from . import system_file_repair
+ with self.contract.database.read_connection() as connection:
+ stored = connection.execute('SELECT result_json FROM ledger_upgrade_repair WHERE id=?',
+ (REVISION,)).fetchone()
+ receipt = json.loads(stored[0]) if stored else None
+ if receipt is None or system_file_repair.RULE in receipt:
+ return
+ earlier = connection.execute(
+ 'SELECT result_json FROM ledger_upgrade_repair WHERE json_type(result_json,?) IS NOT NULL '
+ 'ORDER BY applied_at DESC LIMIT 1', (f'$."{system_file_repair.RULE}"',)).fetchone()
+ if earlier:
+ receipt[system_file_repair.RULE] = json.loads(earlier[0])[system_file_repair.RULE]
+ else:
+ rows = system_file_repair.collect(connection)
+ if earlier:
+ self._save_receipt(receipt)
+ return
+ backup = None
+ if rows:
+ backup = self.contract.db_path.with_name(f'ledger.pre-upgrade-{time.time_ns()}.db')
+ sqlite_backup(self.contract.db_path, backup)
+ with self.contract.database.write_transaction() as connection:
+ changed = system_file_repair.apply(connection, rows, time.time())
+ receipt[system_file_repair.RULE] = {'isolated': changed, 'backup': str(backup) if backup else None}
+ connection.execute('UPDATE ledger_upgrade_repair SET result_json=? WHERE id=?',
+ (json.dumps(receipt, ensure_ascii=False), REVISION))
+ atomic_write_text(self.receipt, json.dumps(receipt, ensure_ascii=False))
def _withdraw_stale_code_evidence(self) -> None:
"""撤掉旧番号下取到的候选与快照(`stale_code_evidence`),被撤的影片按现番号重采。
diff --git a/tests/test_push_discovery.py b/tests/test_push_discovery.py
index 9dfea2b5d..913ea15da 100644
--- a/tests/test_push_discovery.py
+++ b/tests/test_push_discovery.py
@@ -196,6 +196,15 @@ class IgnoreAndSourceTests(unittest.TestCase):
self.assertTrue(push.ignored(name))
self.assertFalse(push.ignored("正片.mp4"))
+ def test_system_directories_are_ignored_before_names_are_encoded(self):
+ for path in (r'R:\media\@eaDir\video.mp4@SynoEAStream',
+ '/media/@snapshot/nested/video.mp4', r'R:\media\#RECYCLE\video.mp4',
+ 'R:\\media\\@eaDir\\broken\udcff.mp4@SynoEAStream',
+ r'R:\media\video.mp4@SynoEAStream'):
+ with self.subTest(path=ascii(path)):
+ self.assertTrue(push.ignored(path))
+ self.assertFalse(push.ignored(r'R:\media\scene@home\video.mp4'))
+
def test_only_loopback_and_private_addresses_are_accepted(self):
# 局域网那一档用 RFC 5737 的文档网段举例:`ipaddress` 把它判成 private,
# 正是 `allowed_source` 看的那一位,写谁家的真实网段都不必要(`test_repo_hygiene`)。
@@ -349,6 +358,27 @@ class _ServiceCase(unittest.TestCase):
class IngestPathTests(_ServiceCase):
+ def test_system_files_never_enter_any_push_registration_path(self):
+ service = self.service()
+ path = self.ledger('local', '@eaDir', 'broken\udcff.mp4@SynoEAStream')
+ with mock.patch.object(service.queue, 'submit') as submit, \
+ mock.patch.object(service, '_ingest') as ingest:
+ self.assertFalse(service.submit('local', path))
+ service.submit_move('local', self.ledger('local', 'old.mp4'), path)
+ service.submit_move('local', path, self.ledger('local', '@eaDir'), directory=True)
+ self.assertFalse(service.ingest_now('local', path))
+ submit.assert_not_called()
+ ingest.assert_not_called()
+
+ def test_direct_ingest_does_not_register_system_directory_contents(self):
+ file = self.media / '@eaDir/video.mp4@SynoEAStream'
+ file.parent.mkdir()
+ file.write_bytes(b'metadata')
+ result = scan.ingest_path(self.db, 'local', self.ledger('local', '@eaDir', file.name),
+ declared_roots=self.declared, mounts=self.mounts)
+ self.assertTrue(result.sidecar)
+ self.assertEqual(self.rows(), {})
+
def test_move_event_keeps_upgrade_existing_asset_without_an_identity_cache(self):
from contextlib import closing
old = self.media / 'old.mp4'
diff --git a/tests/test_upgrade_repairs.py b/tests/test_upgrade_repairs.py
index 779c6396e..b933e27f3 100644
--- a/tests/test_upgrade_repairs.py
+++ b/tests/test_upgrade_repairs.py
@@ -15,6 +15,39 @@ from support.ledger import fresh_ledger
class UpgradeRepairTests(unittest.TestCase):
+ def test_old_system_metadata_is_isolated_once_without_deleting_files_or_records(self):
+ from peach import system_file_repair
+ file = self.root / '@eaDir/video.mp4@SynoEAStream'
+ file.parent.mkdir()
+ file.write_bytes(b'metadata')
+ rows = [(1, str(file), None), (2, r'R:\media\@eaDir\kept@SynoEAStream', 5),
+ (3, r'R:\media\normal.txt', None)]
+ with self.database.write_transaction() as connection:
+ for key, path, rating in rows:
+ connection.execute("INSERT INTO asset(id,location,path,name,medium,rating) "
+ "VALUES(?,'local',?,'metadata','other',?)", (key, path, rating))
+ connection.execute("INSERT INTO asset_tag(asset_id,tag,source) VALUES(1,'retained','test')")
+ self.service.prepare()
+ self.service.prepare()
+ with self.database.read_connection() as connection:
+ self.assertEqual([tuple(row) for row in connection.execute('SELECT id,disposal,rating FROM asset')],
+ [(1, 'trash', None), (2, None, 5), (3, None, None)])
+ self.assertEqual(connection.execute('SELECT count(*) FROM asset_tag WHERE asset_id=1').fetchone()[0], 1)
+ self.assertEqual(file.read_bytes(), b'metadata')
+ receipt = upgrade_repairs._read(self.service.receipt)[system_file_repair.RULE]
+ self.assertEqual(receipt['isolated'], [dict(id=1, path=str(file), disposal=None,
+ disposal_reason=None, feedback_at=None)])
+ self.assertTrue(Path(receipt['backup']).is_file())
+ with self.database.write_transaction() as connection:
+ connection.execute('UPDATE asset SET disposal=NULL,disposal_reason=NULL WHERE id=1')
+ self.service.prepare()
+ with self.database.read_connection() as connection:
+ self.assertIsNone(connection.execute('SELECT disposal FROM asset WHERE id=1').fetchone()[0])
+ with patch.object(upgrade_repairs, 'REVISION', upgrade_repairs.REVISION + '-next'):
+ self.service.prepare()
+ with self.database.read_connection() as connection:
+ self.assertIsNone(connection.execute('SELECT disposal FROM asset WHERE id=1').fetchone()[0])
+
def test_javdb_upgrade_requeues_legacy_evidence_and_keeps_manual_values(self):
from peach import artifact_rules, upgrade_rules
self.asset(1, code='EXM-001', duration=1)