Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(storage): 映射回收站路径并保护不可读媒体

simon156170cd17975f2c
diff --git a/src/peach/web_batch.py b/src/peach/web_batch.py
index 371dcced1..bbbae1940 100644
--- a/src/peach/web_batch.py
+++ b/src/peach/web_batch.py
@@ -11,6 +11,7 @@ from __future__ import annotations
import hashlib
import json
import os
+import stat
import shutil
import tempfile
import time
@@ -296,23 +297,50 @@ def purge_assets(connection, rows, *, missing_only: bool = False):
``missing_only`` 删的只是账本行:文件还在(或答不上在不在)的行进 ``blocked``。
"""
purged, blocked, staged, snapshots, parents = [], [], [], [], []
+ online_roots = None
for row in rows:
media = row["path"]
- if media and missing_only:
+ translated = None
+ if media:
+ try:
+ translated = translate_ledger_path(media)
+ except OSError as error:
+ blocked.append({"id": row["id"], "path": media,
+ "reason": error.strerror or str(error)})
+ continue
+ if is_unmapped(translated):
+ blocked.append({"id": row["id"], "path": media,
+ "reason": "媒体路径未映射到可访问的挂载目录"})
+ continue
+ if PureWindowsPath(str(media)).drive:
+ if online_roots is None:
+ online_roots = _online_source_roots()
+ location = row["location"] if "location" in row.keys() else None
+ if not any(within_root(translated, root) and root_online(root)
+ for root in online_roots.get(location, ())):
+ blocked.append({"id": row["id"], "path": media,
+ "reason": "媒体来源未挂载或不可读取"})
+ continue
+ media_stat = None
+ if media:
try:
- present = translate_ledger_path(media).exists()
- reason = "文件仍在盘上"
+ media_stat = translated.stat()
+ except FileNotFoundError:
+ pass
except OSError as error:
- present, reason = True, error.strerror or str(error)
- if present:
- blocked.append({"id": row["id"], "path": media, "reason": reason})
+ blocked.append({"id": row["id"], "path": media,
+ "reason": error.strerror or str(error)})
+ continue
+ if media and missing_only:
+ if media_stat is not None:
+ blocked.append({"id": row["id"], "path": media, "reason": "文件仍在盘上"})
continue
elif media:
- original = Path(media)
+ original = translated
try:
- if original.exists() and not original.is_file():
+ if media_stat is not None and not stat.S_ISREG(media_stat.st_mode):
raise OSError("not a regular file")
- if original.is_file():
+ if media_stat is not None:
quarantine = _quarantine_media(original, location=row['location'] if 'location' in row.keys() else None)
staged.append((original, quarantine))
except OSError as error:
@@ -323,7 +351,7 @@ def purge_assets(connection, rows, *, missing_only: bool = False):
if snapshot:
snapshots.append(Path(snapshot))
if media:
- parents.append(Path(media).parent)
+ parents.append(translated.parent)
purged.append(row["id"])
try:
if purged:
diff --git a/tests/test_guangya_sources.py b/tests/test_guangya_sources.py
index 56e507492..4dd7ff50e 100644
--- a/tests/test_guangya_sources.py
+++ b/tests/test_guangya_sources.py
@@ -22,6 +22,10 @@ class GuangyaSourceTests(unittest.TestCase):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
self.root = Path(self.temporary.name).resolve()
+ declared = patch.object(web_batch, "LOCATION_ROOT_DECLARATIONS",
+ {location: (str(self.root),) for location in MOUNTED_SOURCES})
+ declared.start()
+ self.addCleanup(declared.stop)
def test_cloud_only_setup_roundtrips_windows_and_macos_mounts(self):
for windows, path in ((True, "G:/Movies"), (False, "/Volumes/Guangya/Movies")):
@@ -118,20 +122,19 @@ class GuangyaSourceTests(unittest.TestCase):
with closing(sqlite3.connect(fresh_ledger(self.root))) as connection:
connection.row_factory = sqlite3.Row
for location in MOUNTED_SOURCES:
- for columns in ("id,location,path,snapshot_path", "id,path,snapshot_path"):
- with self.subTest(location=location, columns=columns):
- media = self.root / "film.mp4"
- media.write_bytes(b"media")
- connection.execute("INSERT INTO asset(id,location,path,name,medium) "
- "VALUES(1,?,?,'film.mp4','video')", (location, str(media)))
- result = web_batch.purge_assets(connection,
- connection.execute(f"SELECT {columns} FROM asset").fetchall())
- self.assertEqual((result["purged"], result["blocked"]), (1, []))
- self.assertFalse(media.exists())
- self.assertEqual(result["_staged"][0][1].read_bytes(), b"media")
- self.assertEqual(connection.execute("SELECT count(*) FROM asset").fetchone()[0], 0)
- web_batch._restore_staged_media(result["_staged"])
- self.assertEqual(media.read_bytes(), b"media")
+ with self.subTest(location=location):
+ media = self.root / "film.mp4"
+ media.write_bytes(b"media")
+ connection.execute("INSERT INTO asset(id,location,path,name,medium) "
+ "VALUES(1,?,?,'film.mp4','video')", (location, str(media)))
+ result = web_batch.purge_assets(connection,
+ connection.execute("SELECT id,location,path,snapshot_path FROM asset").fetchall())
+ self.assertEqual((result["purged"], result["blocked"]), (1, []))
+ self.assertFalse(media.exists())
+ self.assertEqual(result["_staged"][0][1].read_bytes(), b"media")
+ self.assertEqual(connection.execute("SELECT count(*) FROM asset").fetchone()[0], 0)
+ web_batch._restore_staged_media(result["_staged"])
+ self.assertEqual(media.read_bytes(), b"media")
def test_mount_write_error_keeps_the_media_and_ledger_row(self):
with closing(sqlite3.connect(fresh_ledger(self.root))) as connection:
diff --git a/tests/test_rm_web.py b/tests/test_rm_web.py
index 11493c08f..5c3152993 100644
--- a/tests/test_rm_web.py
+++ b/tests/test_rm_web.py
@@ -64,6 +64,12 @@ class WebDataTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = str(fresh_ledger(self.tmp.name))
+ declared = mock.patch.object(web_batch, "LOCATION_ROOT_DECLARATIONS", {
+ "local": (str(Path(self.tmp.name).resolve()),),
+ "115": (str(Path(self.tmp.name).resolve()),),
+ })
+ declared.start()
+ self.addCleanup(declared.stop)
con = sqlite3.connect(self.db_path)
con.executemany(
"""INSERT INTO asset(
@@ -1636,6 +1642,95 @@ class WebDataTests(unittest.TestCase):
).fetchone()[0])
con.close()
+ def test_recycle_bin_delete_translates_windows_ledger_path_to_mounted_media(self):
+ source_root = (Path(self.tmp.name) / "mounted-media").resolve()
+ source_root.mkdir()
+ path = source_root / "one.mp4"
+ path.write_bytes(b"test media")
+ ledger_path = r"R:\Media\one.mp4"
+ with closing(sqlite3.connect(self.db_path)) as con:
+ con.execute("UPDATE asset SET path=?,snapshot_path=NULL WHERE id=1", (ledger_path,))
+ con.commit()
+ rm_web.w_feedback(self.contract, {"id": 1, "kind": "dispose"})
+
+ def translate(raw):
+ if str(raw).casefold() == r"r:\media".casefold():
+ return source_root
+ if str(raw).casefold() == ledger_path.casefold():
+ return path
+ return Path(raw)
+
+ with mock.patch.object(web_batch, "LOCATION_ROOT_DECLARATIONS",
+ {"local": (r"R:\Media",)}), \
+ mock.patch.object(web_batch, "translate_ledger_path", side_effect=translate):
+ result = rm_web.w_batch(self.contract, {"ids": [1], "operation": "delete"})
+
+ self.assertEqual((result["purged"], result["blocked"]), (1, []))
+ self.assertFalse(path.exists(), "永久删除要删除路径映射后的实际媒体文件")
+ self.assertIsNone(self.row(1))
+
+ def test_recycle_bin_delete_keeps_windows_asset_when_its_source_is_unavailable(self):
+ source_root = Path(self.tmp.name).resolve() / "offline-media"
+ path = source_root / "one.mp4"
+ path.parent.mkdir()
+ path.write_bytes(b"test media")
+ ledger_path = r"R:\Media\one.mp4"
+ with closing(sqlite3.connect(self.db_path)) as con:
+ con.execute("UPDATE asset SET path=?,snapshot_path=NULL WHERE id=1", (ledger_path,))
+ con.commit()
+ rm_web.w_feedback(self.contract, {"id": 1, "kind": "dispose"})
+
+ def translate(raw):
+ if str(raw).casefold() == r"r:\media".casefold():
+ return source_root / "not-mounted"
+ if str(raw).casefold() == ledger_path.casefold():
+ return path
+ return Path(raw)
+
+ with mock.patch.object(web_batch, "LOCATION_ROOT_DECLARATIONS",
+ {"local": (r"R:\Media",)}), \
+ mock.patch.object(web_batch, "translate_ledger_path", side_effect=translate):
+ result = rm_web.w_batch(self.contract, {"ids": [1], "operation": "delete"})
+
+ self.assertEqual(result["purged"], 0)
+ self.assertEqual([item["id"] for item in result["blocked"]], [1])
+ self.assertTrue(path.is_file())
+ self.assertEqual(self.row(1)["disposal"], "trash")
+
+ def test_unmapped_paths_keep_media_and_rows_for_delete_and_missing_cleanup(self):
+ from peach.platform import UNMAPPED_ROOT
+
+ media = self.stage_media(1)
+ rm_web.w_feedback(self.contract, {"id": 1, "kind": "dispose"})
+ for missing_only in (False, True):
+ with self.subTest(missing_only=missing_only), \
+ mock.patch.object(web_batch, "translate_ledger_path", return_value=UNMAPPED_ROOT / "one.mp4"):
+ with self.contract.write_transaction() as connection:
+ result = web_batch.purge_assets(connection, [self.row(1)], missing_only=missing_only)
+ self.assertEqual(result["purged"], 0)
+ self.assertEqual([item["id"] for item in result["blocked"]], [1])
+ self.assertTrue(media.is_file())
+ self.assertEqual(self.row(1)["disposal"], "trash")
+
+ def test_unreadable_media_keeps_rows_for_delete_and_missing_cleanup(self):
+ media = self.stage_media(1)
+ rm_web.w_feedback(self.contract, {"id": 1, "kind": "dispose"})
+ original_stat = Path.stat
+
+ def stat(path, *args, **kwargs):
+ if path == media:
+ raise PermissionError("media unavailable")
+ return original_stat(path, *args, **kwargs)
+
+ for missing_only in (False, True):
+ with self.subTest(missing_only=missing_only), mock.patch.object(Path, "stat", stat):
+ with self.contract.write_transaction() as connection:
+ result = web_batch.purge_assets(connection, [self.row(1)], missing_only=missing_only)
+ self.assertEqual(result["purged"], 0)
+ self.assertEqual([item["id"] for item in result["blocked"]], [1])
+ self.assertTrue(media.is_file())
+ self.assertEqual(self.row(1)["disposal"], "trash")
+
def test_permanent_delete_removes_newly_empty_parents_but_keeps_the_source_root(self):
source_root = Path(self.tmp.name) / "source"
media = source_root / "creator" / "release" / "junk.url"
diff --git a/tests/test_trash_junk.py b/tests/test_trash_junk.py
index 90395d10a..0f6cd29dc 100644
--- a/tests/test_trash_junk.py
+++ b/tests/test_trash_junk.py
@@ -10,6 +10,7 @@ from unittest import mock
from pathlib import Path
from support.ledger import fresh_ledger
+from peach import web_batch
ROOT = Path(__file__).resolve().parents[1]
@@ -149,7 +150,8 @@ class TrashJunkTests(unittest.TestCase):
self.addCleanup(connection.close)
if candidates is None:
candidates = {int(row["id"]) for row in rows}
- with mock.patch.object(self.trash_junk, "LOCATION_ROOT_DECLARATIONS", {"local": (str(self.root),)}):
+ with mock.patch.object(self.trash_junk, "LOCATION_ROOT_DECLARATIONS", {"local": (str(self.root),)}), \
+ mock.patch.object(web_batch, "LOCATION_ROOT_DECLARATIONS", {"local": (str(self.root),)}):
return self.trash_junk.purge_reviewed(connection, rows, candidates)
def execute(self, sql, *params):
diff --git a/tests/test_web_resource_sync.py b/tests/test_web_resource_sync.py
index ec56e7119..87a17bb46 100644
--- a/tests/test_web_resource_sync.py
+++ b/tests/test_web_resource_sync.py
@@ -12,7 +12,7 @@ import sqlite3
import tempfile
import time
import unittest
-from pathlib import Path
+from pathlib import Path, PureWindowsPath
from unittest import mock
from peach import web_batch
@@ -42,6 +42,8 @@ class PurgeMissingTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.root = Path(self.tmp.name).resolve()
+ self.mount = self.root / "mount"
+ self.mount.mkdir()
self.db_path = str(fresh_ledger(self.root))
con = sqlite3.connect(self.db_path)
sep = chr(92)
@@ -64,12 +66,12 @@ class PurgeMissingTests(unittest.TestCase):
con.close()
self.contract = rm_web.WebContract(Path(self.db_path))
# 只有 001 和 a.mp4 还在盘上;002、003 当作已被手动删除。
- for name in ("001.jpg", "a.mp4"):
- (self.root / name).write_bytes(b"x")
+ for path in (self.mount / "creator/P/001.jpg",
+ self.mount / "creator/V/a.mp4"):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_bytes(b"x")
# 检查会遍历在线来源的目录来找空文件夹;测试目录必须固定在临时目录中,否则会遍历
# 这台机器上真实的 `R:\media` 与网盘挂载。
- self.mount = self.root / "mount"
- self.mount.mkdir()
for module in (rm_sync, web_batch):
declared = mock.patch.object(module, "LOCATION_ROOT_DECLARATIONS",
{"115": (str(self.mount),)})
@@ -84,8 +86,15 @@ class PurgeMissingTests(unittest.TestCase):
self.tmp.cleanup()
def _translate(self, raw):
- """把账本的 Windows 路径映射到临时目录里的同名文件。"""
- return self.root / str(raw).rsplit(chr(92), 1)[-1]
+ """把来源盘符和相对目录映射进临时挂载;Unix 根路径保持原样。"""
+ text = str(raw)
+ windows = PureWindowsPath(text)
+ drive = windows.drive.upper()
+ if drive == "B:":
+ return self.mount.joinpath(*windows.parts[1:])
+ if drive == "R:":
+ return (self.root / "local").joinpath(*windows.parts[1:])
+ return Path(raw)
def _synced(self, online=lambda location: location == "115"):
"""账本路径映射到临时目录;删除那一侧(`web_batch`)要跟着映射,否则它去问真实的 B:。"""
@@ -292,8 +301,8 @@ class PurgeMissingTests(unittest.TestCase):
preview = rm_sync.w_resource_sync_scan(self.contract)
self.assertEqual(preview["missing"], 2)
- # 账本那几行同住一个目录,列一次;来源根为找空文件夹再列一次。
- self.assertEqual(scandir.call_count, 2)
+ # 两个账本目录各列一次,再遍历来源树的根、父目录和两个子目录。
+ self.assertEqual(scandir.call_count, 6)
source = next(row for row in preview["sources"] if row["location"] == "115")
self.assertEqual(source["total"], 4)
self.assertEqual(source["unreadable"], 0)
@@ -362,8 +371,8 @@ class PurgeMissingTests(unittest.TestCase):
self.assertEqual(preview["missing"], 0)
source = next(row for row in preview["sources"] if row["location"] == "115")
- # 账本那几行所在的目录一个,来源根一个:同一个目录只数一次。
- self.assertEqual(source["unreadable"], 2)
+ # 两个账本目录加来源根:同一个目录只数一次。
+ self.assertEqual(source["unreadable"], 3)
self.assertEqual(preview["empty"], 0)
def test_background_sync_polls_then_rechecks_only_missing_candidates(self):
@@ -437,7 +446,7 @@ class PurgeMissingTests(unittest.TestCase):
def test_intact_directory_reports_no_change(self):
for name in ("002.jpg", "003.jpg"):
- (self.root / name).write_bytes(b"x")
+ (self.mount / "creator/P" / name).write_bytes(b"x")
result = self._run()
self.assertEqual(result["removed"], 0)
self.assertEqual(result["checked"], 3)
@@ -471,7 +480,7 @@ class PurgeMissingTests(unittest.TestCase):
`unreadable` 数不到它,所以清理那一侧照名单删就会删掉几百行文件还在的资产。
"""
for name in ("002.jpg", "003.jpg"):
- (self.root / name).write_bytes(b"x")
+ (self.mount / "creator/P" / name).write_bytes(b"x")
with mock.patch.object(rm_sync, "translate_ledger_path", self._translate), \
mock.patch.object(rm_sync, "_missing_resource_ids",
return_value=([1, 2, 3], 0)):