Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

fix(junk): 组合媒体与目录证据识别短广告并保护分卷和来源水印

longmeidaoab9be417201b
diff --git a/docs/REUSE.md b/docs/REUSE.md
index 576eeaad3..d48285ce2 100644
--- a/docs/REUSE.md
+++ b/docs/REUSE.md
@@ -478,6 +478,10 @@ A 盘同父目录改名复用 CloudDrive 官方 `RenameFile`;跨父且改名
推广网页资源目录中的横幅、标志和导航图片由 `web_batch._attachment_junk_reason` 进入复核队列。判据同时要求 `_files`、网址推广套话、域名、导航元素文件名和体积上限;继承的作品番号不豁免网页导航图片,作品图片、字幕、NFO 与普通 `_files` 目录保留。固定命名的推广附件(安装包、宣传视频、二维码卡片、下载站横幅)按扩展名、体积上限与整名收在词库 `resources/junk/promo_attachment_names.json`,由 `web_batch._promo_name_reason` 读取;命中只计复核档 `JUNK_REVIEW_SCORE`,未探测时长视频的推广目录、推广创作者位与体积证据同样只到复核档,这类分数单独凑不到 `trash_junk.py` 默认的自动移入回收站门槛 `AUTO_TRASH_SCORE`,不凭名称移入回收站或删除。永久删除仍走 `trash_junk.py --purge --review-csv` 和 `purge_assets`。
+极短视频附件复用同一垃圾复核评分、`promo_free_key` 的名称比较和 `release_code_from_filename` 的自身番号识别。发行目录须有与目录同名、至少 20 分钟且至少 120 MiB 的活动物理正片;附件须有有效测量值、不超过 30 秒与 32 MiB,正片至少为附件时长的 40 倍、体积的 10 倍。同目录至少 3 条此类附件,或相同体积、毫秒时长与有效分辨率的附件分布在至少 3 个发行目录,才计结构证据。真实创作者归属、自身番号、分卷、独立短动画与普通混合目录受保护;目录继承的番号不能证明附件身份。结构证据只够复核,短、小或同测量值都不能单独证明是广告,默认自动隔离线不接收仅由这些证据凑分的候选。查询只读账本,不读取云盘媒体,不新增依赖。
+
+带站点水印的文件名先核对自身番号与分卷标记,明确分卷排除在垃圾队列外;「方括号来源站名 + 序号」视频只计复核证据,避免把短片的打包水印当作广告。画面取证可复用已有 FFmpeg 抽帧与 `scripts/windows_ocr.ps1`;本机复核先使用缓存缩略图。现有 `opencv-python-headless==5.0.0.93`(Apache-2.0,vision 可选组)提供 [QRCodeDetector](https://docs.opencv.org/4.7.0/de/dc3/classcv_1_1QRCodeDetector.html),只能定位或解码二维码,不能证明整片是广告;[Windows OcrEngine](https://learn.microsoft.com/en-us/uwp/api/windows.media.ocr.ocrengine.trycreatefromlanguage) 依赖 Windows 与已安装的识别语言。两者均不放进列表请求的同步评分路径,避免平台依赖、图片读取及文字识别失败影响复核队列;内容结论写入仓库外的逐文件复核产物。
+
普通作品图片和 `images.rar` 保留。CloudDrive WebDAV 的小文件删除复用原生 `Path.unlink`:同目录隔离改名报 WinError 50 时,最多 1 MiB 的文件先用标准库 `shutil.copyfile` 完整备份到 `generated/purge-staging/`,记录原路径、大小和 SHA256,再删除。数据库失败时恢复备份,提交后清退;大文件继续报告阻塞。该路径没有新增依赖,备份和恢复只由 `purge_assets` 管理。
英文媒体文件名分词复用 `wordninja-enhanced==3.2.0`(MIT,2026-09-04 发布,Python ≥3.9,纯 Python,wheel 11.4 MB,无传递依赖)。源码经 `opensrc path pypi:[email protected]` 读取;原版 wordninja 的发布较旧,增强版提供标点保留和可补充词典。`space_media_names.py` 先出逐文件复核 CSV,只插入空格,保留序号、字符与扩展名,不推断人物或出处;指定目录内的真实文件核验后复用 `organize.apply_plan`、SQLite 备份及回滚日志。领域词汇在 `english_filename_words.txt`(置顶加权,条目宜少:每多一条都会压低 `a`、`of` 等高频词),常被误当成单词的拼接串在 `english_filename_nonwords.txt` 里移出词频表。撇号缩写、全大写缩写、域名与站点编号、十六进制 viewkey 不拆;西里尔形近字母按拉丁字母算切点、原字符保留。已执行批次用 `--recompute-batch <日志>` 按原名重算出复核 CSV,再走 `--apply --review-csv`。
diff --git a/src/peach/web_batch.py b/src/peach/web_batch.py
index de71fbeb0..109a0259f 100644
--- a/src/peach/web_batch.py
+++ b/src/peach/web_batch.py
@@ -23,7 +23,10 @@ from functools import cache
from pathlib import Path, PureWindowsPath
from typing import Sequence
-from .catalog_rules import duration_clusters, is_jav_code, normalise_code_key
+from .catalog_rules import (
+ duration_clusters, is_jav_code, normalise_code_key, promo_free_key,
+ release_code_from_filename,
+)
from .config import GENERATED_DIR, LOCATION_ROOT_DECLARATIONS
from .field_owners import USER_MANUAL, write_owned_fields
from .media_sources import SCANNABLE_LOCATIONS
@@ -156,6 +159,12 @@ JUNK_REVIEW_SCORE = 40
#: `scripts/trash_junk.py --apply` 默认的自动移入回收站分数线。只能复核的证据
#: (固定命名、未探测时长视频的目录与体积证据)单独凑不到这一档。
AUTO_TRASH_SCORE = 60
+#: 极短附件的结构证据只进入复核:目录中须有同名长正片,且附件相对它足够短、小。
+SHORT_ATTACHMENT_SECONDS = 30
+SHORT_ATTACHMENT_BYTES = 32 * 1024**2
+SHORT_ATTACHMENT_CLUSTER = 3
+#: 「来源站名 + 序号」也用于正常短片;站点水印不能成为自动隔离依据。
+NUMBERED_SOURCE_VIDEO = re.compile(r"^(?:\[[^\]]+\]|【[^】]+】)[\s._-]*\d+(?:[\s._-]*\(\d+\))?$")
PAGE_COMPONENT_SUFFIXES = frozenset({".js", ".css", ".aspx"})
PAGE_CHROME_IMAGE = re.compile(r"^(?:banner(?:_[bs])?|1000x90(?:yunding)?|count\d*|logo|lan\d*|you|zuo|zuoxia)$", re.I)
PAGE_PROMO_CLAIM = re.compile(r"(?:记住|記住).{0,12}(?:网址|網址)|(?:网址|網址).{0,8}(?:发布|發布)")
@@ -250,8 +259,17 @@ def _junk_rows(connection):
(*SCANNABLE_LOCATIONS, JUNK_VIDEO_MAX_BYTES)).fetchall()
# NFO 是媒体资料边车,不是等待清理的物理内容;即使名字或所在目录带推广词,
# 也应由资料读取报告解析问题,不能进入会把文件移入回收站的垃圾队列。
- return [row for row in rows if PureWindowsPath(
- row["name"] or row["path"] or "").suffix.casefold() not in MEDIA_SIDECAR_SUFFIXES]
+ return [row for row in rows if _junk_file_eligible(row)]
+
+
+def _junk_file_eligible(row) -> bool:
+ """资料边车和文件自身带番号的分卷不属于垃圾附件。"""
+ item = PureWindowsPath(row["name"] or row["path"] or "")
+ if item.suffix.casefold() in MEDIA_SIDECAR_SUFFIXES:
+ return False
+ identity = PART_MARK.sub("", PROMO_DOMAIN.sub("", item.stem)).strip("@._- ")
+ return not (row["medium"] == "video" and PART_MARK.search(item.stem)
+ and release_code_from_filename(identity))
def q_ads(contract: WebContract, limit=200, offset=0, kind="", status="pending"):
@@ -382,8 +400,68 @@ def _attachment_junk_reason(suffix: str, path: str, size: int) -> str:
return ""
+def _short_attachment(row) -> bool:
+ """有测量值、没有自身作品身份或分卷标记的极短视频。"""
+ stem = PureWindowsPath(row["name"] or row["path"] or "").stem
+ owner = row["creator"] or ""
+ return (row["medium"] == "video"
+ and 0 < (row["duration"] or 0) <= SHORT_ATTACHMENT_SECONDS
+ and 0 < (row["size"] or 0) <= SHORT_ATTACHMENT_BYTES
+ and not (owner and not AD_DOMAIN.search(owner))
+ and not PART_MARK.search(stem)
+ and not release_code_from_filename(stem))
+
+
+def _release_main_videos(connection) -> dict[str, list[tuple[float, int]]]:
+ """按活动物理文件建立发行目录的长正片索引;普通混合目录不算发行目录。"""
+ mains: dict[str, list[tuple[float, int]]] = {}
+ marks = ",".join("?" for _ in SCANNABLE_LOCATIONS)
+ for path, duration, size in connection.execute(
+ "SELECT path,duration,size FROM asset WHERE disposal IS NULL AND medium='video' "
+ f"AND location IN ({marks}) AND duration>=1200 AND size>=? AND path IS NOT NULL",
+ (*SCANNABLE_LOCATIONS, JUNK_VIDEO_MAX_BYTES)):
+ item = PureWindowsPath(path)
+ key = promo_free_key(item.stem)
+ if key and key == promo_free_key(item.parent.name):
+ mains.setdefault(str(item.parent).casefold(), []).append((duration, size))
+ return mains
+
+
+def _short_attachment_reasons(connection, rows) -> dict[int, str]:
+ """从正片与附件的量级关系、同目录成群和跨发行目录扩散生成复核证据。
+
+ 不读媒体,也不按题材、游戏名或推广词命中。相同测量值只是扩散线索,不能证明内容相同。
+ """
+ mains = _release_main_videos(connection)
+ folders: dict[str, set[str]] = {}
+ signatures: dict[tuple, set[str]] = {}
+ candidates = []
+ for row in rows:
+ if not _short_attachment(row):
+ continue
+ path = PureWindowsPath(row["path"] or "")
+ folder = str(path.parent).casefold()
+ if not any(duration >= row["duration"] * 40 and size >= row["size"] * 10
+ for duration, size in mains.get(folder, ())):
+ continue
+ signature = (row["size"], round(row["duration"], 3), row["width"], row["height"])
+ folders.setdefault(folder, set()).add(str(path).casefold())
+ if (row["width"] or 0) > 0 and (row["height"] or 0) > 0:
+ signatures.setdefault(signature, set()).add(folder)
+ candidates.append((row["id"], folder, signature))
+ reasons = {}
+ for asset_id, folder, signature in candidates:
+ count = len(folders[folder])
+ spread = len(signatures.get(signature, ()))
+ if count >= SHORT_ATTACHMENT_CLUSTER:
+ reasons[asset_id] = f"同目录长正片旁有 {count} 条极短小附件,须核验内容"
+ elif spread >= SHORT_ATTACHMENT_CLUSTER:
+ reasons[asset_id] = f"同体积、时长与分辨率的极短附件散布在 {spread} 个发行目录,须核验内容"
+ return reasons
+
+
def _video_evidence(row, stem: str, longer: dict, promo_dir: bool,
- directory_points: int) -> tuple[int, int, list[str]]:
+ directory_points: int, attachment_reason: str = "") -> tuple[int, int, list[str]]:
"""视频自己的证据:同番号完整版、时长、体积与推广目录。
返回(分数,其中只够复核的分数,理由)。时长没探测到的视频可能就是正片:
@@ -409,7 +487,13 @@ def _video_evidence(row, stem: str, longer: dict, promo_dir: bool,
# 要再叠一条时长或体积证据才到门槛。
score += 30; context += 30; why.append("住在推广目录")
probed = duration is not None and duration > 0
- return score, 0 if probed else context, why
+ review_only = 0 if probed else context
+ if NUMBERED_SOURCE_VIDEO.fullmatch(stem) and PROMO_DOMAIN.search(stem):
+ review_only += AUTO_TRASH_SCORE
+ why.append("编号视频可能只有来源水印,须核验内容")
+ if attachment_reason:
+ score += JUNK_REVIEW_SCORE; review_only += JUNK_REVIEW_SCORE; why.append(attachment_reason)
+ return score, review_only, why
def _scored_junk(contract: WebContract) -> tuple[list[dict], frozenset[int]]:
@@ -429,6 +513,7 @@ def _scored_junk(contract: WebContract) -> tuple[list[dict], frozenset[int]]:
)]
dismissed_ids = {int(key) for key in dismissed_keys if key.isdigit()}
folder_assets, folder_videos = _folder_index(c)
+ short_attachments = _short_attachment_reasons(c, rows)
# 每个目录里挂着推广名的候选有多少个;插页判据要看它有没有同伙。
promo_neighbours = _promo_neighbour_counts(rows)
out = []
@@ -516,7 +601,8 @@ def _scored_junk(contract: WebContract) -> tuple[list[dict], frozenset[int]]:
elif promo_dir:
s += 50; why.append("住在推广目录")
if d.get("medium") == "video":
- points, weak, reasons = _video_evidence(d, nm, longer, promo_dir, directory_points)
+ points, weak, reasons = _video_evidence(
+ d, nm, longer, promo_dir, directory_points, short_attachments.get(d["id"], ""))
s += points; review_only += weak; why.extend(reasons)
# 有真实创作者归属、且名字剥完仍有实质描述的,是被打了水印的正片,不是广告。
if real_owner and residue >= 14 and not (attachment_reason or name_reason):
diff --git a/tests/test_ad_judgement.py b/tests/test_ad_judgement.py
index 8016c1b17..30e5af826 100644
--- a/tests/test_ad_judgement.py
+++ b/tests/test_ad_judgement.py
@@ -7,6 +7,7 @@ import sqlite3
import tempfile
import unittest
from contextlib import closing
+from pathlib import Path
from unittest import mock
from peach import web_batch
@@ -156,7 +157,7 @@ class ResourceJunkQueueTests(unittest.TestCase):
def setUp(self):
self.temporary = tempfile.TemporaryDirectory()
self.addCleanup(self.temporary.cleanup)
- self.db_path = fresh_ledger(self.temporary.name)
+ self.db_path = fresh_ledger(Path(self.temporary.name).resolve())
self.contract = WebContract(self.db_path)
def add(self, asset_id, location, path, medium, size=1000, duration=None,
@@ -219,6 +220,104 @@ class ResourceJunkQueueTests(unittest.TestCase):
self.assertLess(row["score"], web_batch.AUTO_TRASH_SCORE)
self.assertEqual(web_batch.automatic_junk_candidates(self.contract), [])
+ def release_with_shorts(self, base, first_id=300, names=None, **short_values):
+ self.add(first_id, "115", base + "\\" + base.rsplit("\\", 1)[-1] + ".mp4",
+ "video", 1103584330, 3264.96)
+ for offset, name in enumerate(names or ("附件甲", "附件乙", "附件丙"), 1):
+ values = dict(size=24880490, duration=19.319002)
+ values.update(short_values)
+ self.add(first_id + offset, "115", base + "\\" + name + ".mp4", "video", **values)
+
+ def test_short_attachment_cluster_is_reviewed_without_promo_words(self):
+ self.release_with_shorts(r"B:\发行物\正片标题", names=(
+ "海贼王3D版 成人欲海狂潮 黄 游", "附件甲", "附件乙", "附件丙", "附件丁", "附件戊"))
+ items = q_ads(self.contract)["items"]
+ self.assertEqual({row["id"] for row in items}, set(range(301, 307)))
+ self.assertTrue(all("6 条极短小附件" in row["why"] for row in items))
+ self.assertTrue(all(web_batch.JUNK_REVIEW_SCORE <= row["score"] < web_batch.AUTO_TRASH_SCORE
+ for row in items))
+ self.assertEqual(web_batch.automatic_junk_candidates(self.contract), [])
+
+ def test_a_pair_of_shorts_and_standalone_animation_loops_stay_out(self):
+ self.release_with_shorts(r"B:\发行物\正片标题", names=("附件甲", "附件乙"))
+ for asset_id in range(310, 316):
+ self.add(asset_id, "115", rf"B:\画师作品集\动画{asset_id}.mp4", "video", 24880490, 19.319002)
+ self.assertEqual(q_ads(self.contract)["items"], [])
+
+ def test_short_attachment_identity_and_measurements_must_be_eligible(self):
+ cases = (
+ ({"duration": None}, None), ({"duration": 0}, None), ({"duration": -1}, None),
+ ({"duration": 31}, None), ({"size": 0}, None),
+ ({"size": 33 * 1024**2}, None), ({"creator": "示例画师"}, None),
+ ({}, ("作品 CD1", "作品 part2", "作品 分卷")),
+ ({}, ("ABW-153", "ABW-154", "ABW-155")),
+ )
+ for index, (values, names) in enumerate(cases):
+ self.release_with_shorts(rf"B:\发行物\正片{index}", 400 + index * 10,
+ names=names, **values)
+ self.assertEqual(q_ads(self.contract)["items"], [])
+
+ def test_short_attachments_can_have_inherited_directory_codes(self):
+ self.release_with_shorts(r"B:\发行物\ABW-153", code="ABW-153")
+ self.assertEqual({row["id"] for row in q_ads(self.contract)["items"]}, {301, 302, 303})
+
+ def test_mixed_folder_or_inactive_main_cannot_support_attachment_evidence(self):
+ for index, main_values in enumerate((
+ {"path": r"B:\发行物\正片0\别的标题.mp4"}, {"disposal": "trash"}, {"location": "online"},
+ {"duration": 1100}, {"size": 40 * 1024**2})):
+ main_id = 500 + index * 10
+ self.release_with_shorts(rf"B:\发行物\正片{index}", main_id)
+ with closing(sqlite3.connect(self.db_path)) as connection:
+ for column, value in main_values.items():
+ connection.execute(f"UPDATE asset SET {column}=? WHERE id=?", (value, main_id))
+ connection.commit()
+ self.assertEqual(q_ads(self.contract)["items"], [])
+
+ def test_cross_release_spread_uses_measurements_with_different_names(self):
+ for index in range(3):
+ self.release_with_shorts(rf"B:\发行物\正片{index}", 600 + index * 10,
+ names=(f"无推广字样的名字{index}",))
+ with closing(sqlite3.connect(self.db_path)) as connection:
+ connection.execute("UPDATE asset SET width=1920,height=1080")
+ connection.commit()
+ items = q_ads(self.contract)["items"]
+ self.assertEqual({row["id"] for row in items}, {601, 611, 621})
+ self.assertTrue(all("3 个发行目录" in row["why"] for row in items))
+ self.assertEqual(web_batch.automatic_junk_candidates(self.contract), [])
+
+ def test_cross_release_spread_requires_full_matching_measurements(self):
+ for index in range(3):
+ self.release_with_shorts(rf"B:\发行物\正片{index}", 600 + index * 10, names=("附件",))
+ self.assertEqual(q_ads(self.contract)["items"], [])
+ with closing(sqlite3.connect(self.db_path)) as connection:
+ connection.execute("UPDATE asset SET width=1920,height=1080")
+ connection.execute("UPDATE asset SET duration=20 WHERE id=621")
+ connection.commit()
+ self.assertEqual(q_ads(self.contract)["items"], [])
+
+ def test_dismissed_short_attachment_can_be_reconsidered(self):
+ self.release_with_shorts(r"B:\发行物\正片标题")
+ w_batch(self.contract, {"ids": [301], "operation": "dismiss-junk"})
+ self.assertEqual({row["id"] for row in q_ads(self.contract)["items"]}, {302, 303})
+ w_batch(self.contract, {"ids": [301], "operation": "reconsider-junk"})
+ self.assertEqual({row["id"] for row in q_ads(self.contract)["items"]}, {301, 302, 303})
+
+ def test_domain_watermarked_release_volumes_are_not_junk(self):
+ names = ("[email protected]", "www.example.com_ABW-153-part1.mp4",
+ "FC2-PPV-1234567-2.mp4", "ABW-153 (3).mp4")
+ for asset_id, name in enumerate(names, 700):
+ self.add(asset_id, "115", "B:\\发行物\\" + name, "video", 10 * 1024**2, 400)
+ self.assertEqual(q_ads(self.contract)["items"], [])
+ self.assertEqual(web_batch.automatic_junk_candidates(self.contract), [])
+
+ def test_numbered_source_watermarks_only_enter_review(self):
+ for asset_id, name in enumerate(("【来源站example.com】1", "[example.com]84 (2)"), 710):
+ self.add(asset_id, "115", "B:\\短片\\" + name + ".mp4", "video", 5 * 1024**2, 10)
+ items = q_ads(self.contract)["items"]
+ self.assertEqual({row["id"] for row in items}, {710, 711})
+ self.assertTrue(all(row["score"] < web_batch.AUTO_TRASH_SCORE for row in items))
+ self.assertEqual(web_batch.automatic_junk_candidates(self.contract), [])
+
def test_non_video_trash_items_remain_visible_and_recoverable(self):
self.add(10, "115", r"B:\番号\广告图.jpg", "image", disposal="trash")
self.add(11, "115", r"B:\番号\备用网址.url", "other", disposal="trash")