Peach: Local-first video metadata scraper and media manager.
fix(storage): 映射回收站路径并保护不可读媒体
simon156170cd17975f2c
diff --git a/src/peach/web_batch.py b/src/peach/web_batch.pyindex 371dcced1..bbbae1940 100644--- a/src/peach/web_batch.py+++ b/src/peach/web_batch.py@@ -11,6 +11,7 @@ from __future__ import annotationsimport hashlibimport jsonimport os+import statimport shutilimport tempfileimport time@@ -296,23 +297,50 @@ def purge_assets(connection, rows, *, missing_only: bool = False):``missing_only`` 删的只是账本行:文件还在(或答不上在不在)的行进 ``blocked``。"""purged, blocked, staged, snapshots, parents = [], [], [], [], []+ online_roots = Nonefor row in rows:media = row["path"]- if media and missing_only:+ translated = None+ if media:+ try:+ translated = translate_ledger_path(media)+ except OSError as error:+ blocked.append({"id": row["id"], "path": media,+ "reason": error.strerror or str(error)})+ continue+ if is_unmapped(translated):+ blocked.append({"id": row["id"], "path": media,+ "reason": "媒体路径未映射到可访问的挂载目录"})+ continue+ if PureWindowsPath(str(media)).drive:+ if online_roots is None:+ online_roots = _online_source_roots()+ location = row["location"] if "location" in row.keys() else None+ if not any(within_root(translated, root) and root_online(root)+ for root in online_roots.get(location, ())):+ blocked.append({"id": row["id"], "path": media,+ "reason": "媒体来源未挂载或不可读取"})+ continue+ media_stat = None+ if media:try:- present = translate_ledger_path(media).exists()- reason = "文件仍在盘上"+ media_stat = translated.stat()+ except FileNotFoundError:+ passexcept OSError as error:- present, reason = True, error.strerror or str(error)- if present:- blocked.append({"id": row["id"], "path": media, "reason": reason})+ blocked.append({"id": row["id"], "path": media,+ "reason": error.strerror or str(error)})+ continue+ if media and missing_only:+ if media_stat is not None:+ blocked.append({"id": row["id"], "path": media, "reason": "文件仍在盘上"})continueelif media:- original = Path(media)+ original = translatedtry:- if original.exists() and not original.is_file():+ if media_stat is not None and not stat.S_ISREG(media_stat.st_mode):raise OSError("not a regular file")- if original.is_file():+ if media_stat is not None:quarantine = _quarantine_media(original, location=row['location'] if 'location' in row.keys() else None)staged.append((original, quarantine))except OSError as error:@@ -323,7 +351,7 @@ def purge_assets(connection, rows, *, missing_only: bool = False):if snapshot:snapshots.append(Path(snapshot))if media:- parents.append(Path(media).parent)+ parents.append(translated.parent)purged.append(row["id"])try:if purged:diff --git a/tests/test_guangya_sources.py b/tests/test_guangya_sources.pyindex 56e507492..4dd7ff50e 100644--- a/tests/test_guangya_sources.py+++ b/tests/test_guangya_sources.py@@ -22,6 +22,10 @@ class GuangyaSourceTests(unittest.TestCase):self.temporary = tempfile.TemporaryDirectory()self.addCleanup(self.temporary.cleanup)self.root = Path(self.temporary.name).resolve()+ declared = patch.object(web_batch, "LOCATION_ROOT_DECLARATIONS",+ {location: (str(self.root),) for location in MOUNTED_SOURCES})+ declared.start()+ self.addCleanup(declared.stop)def test_cloud_only_setup_roundtrips_windows_and_macos_mounts(self):for windows, path in ((True, "G:/Movies"), (False, "/Volumes/Guangya/Movies")):@@ -118,20 +122,19 @@ class GuangyaSourceTests(unittest.TestCase):with closing(sqlite3.connect(fresh_ledger(self.root))) as connection:connection.row_factory = sqlite3.Rowfor location in MOUNTED_SOURCES:- for columns in ("id,location,path,snapshot_path", "id,path,snapshot_path"):- with self.subTest(location=location, columns=columns):- media = self.root / "film.mp4"- media.write_bytes(b"media")- connection.execute("INSERT INTO asset(id,location,path,name,medium) "- "VALUES(1,?,?,'film.mp4','video')", (location, str(media)))- result = web_batch.purge_assets(connection,- connection.execute(f"SELECT {columns} FROM asset").fetchall())- self.assertEqual((result["purged"], result["blocked"]), (1, []))- self.assertFalse(media.exists())- self.assertEqual(result["_staged"][0][1].read_bytes(), b"media")- self.assertEqual(connection.execute("SELECT count(*) FROM asset").fetchone()[0], 0)- web_batch._restore_staged_media(result["_staged"])- self.assertEqual(media.read_bytes(), b"media")+ with self.subTest(location=location):+ media = self.root / "film.mp4"+ media.write_bytes(b"media")+ connection.execute("INSERT INTO asset(id,location,path,name,medium) "+ "VALUES(1,?,?,'film.mp4','video')", (location, str(media)))+ result = web_batch.purge_assets(connection,+ connection.execute("SELECT id,location,path,snapshot_path FROM asset").fetchall())+ self.assertEqual((result["purged"], result["blocked"]), (1, []))+ self.assertFalse(media.exists())+ self.assertEqual(result["_staged"][0][1].read_bytes(), b"media")+ self.assertEqual(connection.execute("SELECT count(*) FROM asset").fetchone()[0], 0)+ web_batch._restore_staged_media(result["_staged"])+ self.assertEqual(media.read_bytes(), b"media")def test_mount_write_error_keeps_the_media_and_ledger_row(self):with closing(sqlite3.connect(fresh_ledger(self.root))) as connection:diff --git a/tests/test_rm_web.py b/tests/test_rm_web.pyindex 11493c08f..5c3152993 100644--- a/tests/test_rm_web.py+++ b/tests/test_rm_web.py@@ -64,6 +64,12 @@ class WebDataTests(unittest.TestCase):def setUp(self):self.tmp = tempfile.TemporaryDirectory()self.db_path = str(fresh_ledger(self.tmp.name))+ declared = mock.patch.object(web_batch, "LOCATION_ROOT_DECLARATIONS", {+ "local": (str(Path(self.tmp.name).resolve()),),+ "115": (str(Path(self.tmp.name).resolve()),),+ })+ declared.start()+ self.addCleanup(declared.stop)con = sqlite3.connect(self.db_path)con.executemany("""INSERT INTO asset(@@ -1636,6 +1642,95 @@ class WebDataTests(unittest.TestCase):).fetchone()[0])con.close()+ def test_recycle_bin_delete_translates_windows_ledger_path_to_mounted_media(self):+ source_root = (Path(self.tmp.name) / "mounted-media").resolve()+ source_root.mkdir()+ path = source_root / "one.mp4"+ path.write_bytes(b"test media")+ ledger_path = r"R:\Media\one.mp4"+ with closing(sqlite3.connect(self.db_path)) as con:+ con.execute("UPDATE asset SET path=?,snapshot_path=NULL WHERE id=1", (ledger_path,))+ con.commit()+ rm_web.w_feedback(self.contract, {"id": 1, "kind": "dispose"})++ def translate(raw):+ if str(raw).casefold() == r"r:\media".casefold():+ return source_root+ if str(raw).casefold() == ledger_path.casefold():+ return path+ return Path(raw)++ with mock.patch.object(web_batch, "LOCATION_ROOT_DECLARATIONS",+ {"local": (r"R:\Media",)}), \+ mock.patch.object(web_batch, "translate_ledger_path", side_effect=translate):+ result = rm_web.w_batch(self.contract, {"ids": [1], "operation": "delete"})++ self.assertEqual((result["purged"], result["blocked"]), (1, []))+ self.assertFalse(path.exists(), "永久删除要删除路径映射后的实际媒体文件")+ self.assertIsNone(self.row(1))++ def test_recycle_bin_delete_keeps_windows_asset_when_its_source_is_unavailable(self):+ source_root = Path(self.tmp.name).resolve() / "offline-media"+ path = source_root / "one.mp4"+ path.parent.mkdir()+ path.write_bytes(b"test media")+ ledger_path = r"R:\Media\one.mp4"+ with closing(sqlite3.connect(self.db_path)) as con:+ con.execute("UPDATE asset SET path=?,snapshot_path=NULL WHERE id=1", (ledger_path,))+ con.commit()+ rm_web.w_feedback(self.contract, {"id": 1, "kind": "dispose"})++ def translate(raw):+ if str(raw).casefold() == r"r:\media".casefold():+ return source_root / "not-mounted"+ if str(raw).casefold() == ledger_path.casefold():+ return path+ return Path(raw)++ with mock.patch.object(web_batch, "LOCATION_ROOT_DECLARATIONS",+ {"local": (r"R:\Media",)}), \+ mock.patch.object(web_batch, "translate_ledger_path", side_effect=translate):+ result = rm_web.w_batch(self.contract, {"ids": [1], "operation": "delete"})++ self.assertEqual(result["purged"], 0)+ self.assertEqual([item["id"] for item in result["blocked"]], [1])+ self.assertTrue(path.is_file())+ self.assertEqual(self.row(1)["disposal"], "trash")++ def test_unmapped_paths_keep_media_and_rows_for_delete_and_missing_cleanup(self):+ from peach.platform import UNMAPPED_ROOT++ media = self.stage_media(1)+ rm_web.w_feedback(self.contract, {"id": 1, "kind": "dispose"})+ for missing_only in (False, True):+ with self.subTest(missing_only=missing_only), \+ mock.patch.object(web_batch, "translate_ledger_path", return_value=UNMAPPED_ROOT / "one.mp4"):+ with self.contract.write_transaction() as connection:+ result = web_batch.purge_assets(connection, [self.row(1)], missing_only=missing_only)+ self.assertEqual(result["purged"], 0)+ self.assertEqual([item["id"] for item in result["blocked"]], [1])+ self.assertTrue(media.is_file())+ self.assertEqual(self.row(1)["disposal"], "trash")++ def test_unreadable_media_keeps_rows_for_delete_and_missing_cleanup(self):+ media = self.stage_media(1)+ rm_web.w_feedback(self.contract, {"id": 1, "kind": "dispose"})+ original_stat = Path.stat++ def stat(path, *args, **kwargs):+ if path == media:+ raise PermissionError("media unavailable")+ return original_stat(path, *args, **kwargs)++ for missing_only in (False, True):+ with self.subTest(missing_only=missing_only), mock.patch.object(Path, "stat", stat):+ with self.contract.write_transaction() as connection:+ result = web_batch.purge_assets(connection, [self.row(1)], missing_only=missing_only)+ self.assertEqual(result["purged"], 0)+ self.assertEqual([item["id"] for item in result["blocked"]], [1])+ self.assertTrue(media.is_file())+ self.assertEqual(self.row(1)["disposal"], "trash")+def test_permanent_delete_removes_newly_empty_parents_but_keeps_the_source_root(self):source_root = Path(self.tmp.name) / "source"media = source_root / "creator" / "release" / "junk.url"diff --git a/tests/test_trash_junk.py b/tests/test_trash_junk.pyindex 90395d10a..0f6cd29dc 100644--- a/tests/test_trash_junk.py+++ b/tests/test_trash_junk.py@@ -10,6 +10,7 @@ from unittest import mockfrom pathlib import Pathfrom support.ledger import fresh_ledger+from peach import web_batchROOT = Path(__file__).resolve().parents[1]@@ -149,7 +150,8 @@ class TrashJunkTests(unittest.TestCase):self.addCleanup(connection.close)if candidates is None:candidates = {int(row["id"]) for row in rows}- with mock.patch.object(self.trash_junk, "LOCATION_ROOT_DECLARATIONS", {"local": (str(self.root),)}):+ with mock.patch.object(self.trash_junk, "LOCATION_ROOT_DECLARATIONS", {"local": (str(self.root),)}), \+ mock.patch.object(web_batch, "LOCATION_ROOT_DECLARATIONS", {"local": (str(self.root),)}):return self.trash_junk.purge_reviewed(connection, rows, candidates)def execute(self, sql, *params):diff --git a/tests/test_web_resource_sync.py b/tests/test_web_resource_sync.pyindex ec56e7119..87a17bb46 100644--- a/tests/test_web_resource_sync.py+++ b/tests/test_web_resource_sync.py@@ -12,7 +12,7 @@ import sqlite3import tempfileimport timeimport unittest-from pathlib import Path+from pathlib import Path, PureWindowsPathfrom unittest import mockfrom peach import web_batch@@ -42,6 +42,8 @@ class PurgeMissingTests(unittest.TestCase):def setUp(self):self.tmp = tempfile.TemporaryDirectory()self.root = Path(self.tmp.name).resolve()+ self.mount = self.root / "mount"+ self.mount.mkdir()self.db_path = str(fresh_ledger(self.root))con = sqlite3.connect(self.db_path)sep = chr(92)@@ -64,12 +66,12 @@ class PurgeMissingTests(unittest.TestCase):con.close()self.contract = rm_web.WebContract(Path(self.db_path))# 只有 001 和 a.mp4 还在盘上;002、003 当作已被手动删除。- for name in ("001.jpg", "a.mp4"):- (self.root / name).write_bytes(b"x")+ for path in (self.mount / "creator/P/001.jpg",+ self.mount / "creator/V/a.mp4"):+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_bytes(b"x")# 检查会遍历在线来源的目录来找空文件夹;测试目录必须固定在临时目录中,否则会遍历# 这台机器上真实的 `R:\media` 与网盘挂载。- self.mount = self.root / "mount"- self.mount.mkdir()for module in (rm_sync, web_batch):declared = mock.patch.object(module, "LOCATION_ROOT_DECLARATIONS",{"115": (str(self.mount),)})@@ -84,8 +86,15 @@ class PurgeMissingTests(unittest.TestCase):self.tmp.cleanup()def _translate(self, raw):- """把账本的 Windows 路径映射到临时目录里的同名文件。"""- return self.root / str(raw).rsplit(chr(92), 1)[-1]+ """把来源盘符和相对目录映射进临时挂载;Unix 根路径保持原样。"""+ text = str(raw)+ windows = PureWindowsPath(text)+ drive = windows.drive.upper()+ if drive == "B:":+ return self.mount.joinpath(*windows.parts[1:])+ if drive == "R:":+ return (self.root / "local").joinpath(*windows.parts[1:])+ return Path(raw)def _synced(self, online=lambda location: location == "115"):"""账本路径映射到临时目录;删除那一侧(`web_batch`)要跟着映射,否则它去问真实的 B:。"""@@ -292,8 +301,8 @@ class PurgeMissingTests(unittest.TestCase):preview = rm_sync.w_resource_sync_scan(self.contract)self.assertEqual(preview["missing"], 2)- # 账本那几行同住一个目录,列一次;来源根为找空文件夹再列一次。- self.assertEqual(scandir.call_count, 2)+ # 两个账本目录各列一次,再遍历来源树的根、父目录和两个子目录。+ self.assertEqual(scandir.call_count, 6)source = next(row for row in preview["sources"] if row["location"] == "115")self.assertEqual(source["total"], 4)self.assertEqual(source["unreadable"], 0)@@ -362,8 +371,8 @@ class PurgeMissingTests(unittest.TestCase):self.assertEqual(preview["missing"], 0)source = next(row for row in preview["sources"] if row["location"] == "115")- # 账本那几行所在的目录一个,来源根一个:同一个目录只数一次。- self.assertEqual(source["unreadable"], 2)+ # 两个账本目录加来源根:同一个目录只数一次。+ self.assertEqual(source["unreadable"], 3)self.assertEqual(preview["empty"], 0)def test_background_sync_polls_then_rechecks_only_missing_candidates(self):@@ -437,7 +446,7 @@ class PurgeMissingTests(unittest.TestCase):def test_intact_directory_reports_no_change(self):for name in ("002.jpg", "003.jpg"):- (self.root / name).write_bytes(b"x")+ (self.mount / "creator/P" / name).write_bytes(b"x")result = self._run()self.assertEqual(result["removed"], 0)self.assertEqual(result["checked"], 3)@@ -471,7 +480,7 @@ class PurgeMissingTests(unittest.TestCase):`unreadable` 数不到它,所以清理那一侧照名单删就会删掉几百行文件还在的资产。"""for name in ("002.jpg", "003.jpg"):- (self.root / name).write_bytes(b"x")+ (self.mount / "creator/P" / name).write_bytes(b"x")with mock.patch.object(rm_sync, "translate_ledger_path", self._translate), \mock.patch.object(rm_sync, "_missing_resource_ids",return_value=([1, 2, 3], 0)):