Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

12.4 KB
Raw Blame
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""把广告候选移进回收站,或按逐文件复核清单永久删除。

候选来自 `peach.web_batch.q_ads`,与垃圾复核页是同一套评分与判据;这里只按分数
与来源过滤,然后写 `disposal='trash'`——文件不动、账本行保留,清空回收站才真正
删除,误判可以随时从回收站恢复。默认 dry-run 只出清单,`--apply` 必须同时给
`--backup`。

`--purge --review-csv` 只处理清单里 decision=delete 的确认项。每项须带路径、体积、
证据与至少 0.95 的确认置信度;执行时核对文件未变化、来源在线且路径在根内,保护字幕与 NFO。
以下几类整条跳过、不删,并在结果的 `skipped` 里逐条回报:已不在垃圾复核队列里的行、
用户点过「不是垃圾」的行、带个人记录(播放、评分、偏好、稍后看、播放列表等,判据同
`personal_records.record_holders`)的行,以及登记了字幕或盘上有同名字幕、NFO 的行。
删除只走 `purge_assets()`。

用法:
    python scripts/trash_junk.py --min-score 60
    python scripts/trash_junk.py --min-score 60 --apply --backup <落点>
"""
from __future__ import annotations

import argparse
import json
import math
import sys
import time
from pathlib import Path

PROJECT_ROOT = Path(__file__).resolve().parent.parent
SRC_DIR = PROJECT_ROOT / "src"
if str(SRC_DIR) not in sys.path:
    sys.path.insert(0, str(SRC_DIR))

from peach.config import DATABASE_PATH, GENERATED_DIR, LOCATION_ROOT_DECLARATIONS
from peach.media_sources import SCANNABLE_LOCATIONS
from peach.personal_records import record_holders
from peach.platform import root_online, translate_ledger_path, within_root
from peach.review_csv import read_rows, write_rows
from peach.scripting import (
    add_ledger_write_args, counts_of, open_for_write, open_readonly, verify_after_write,
)
from peach.web_batch import (
    AUTO_TRASH_SCORE, MEDIA_SIDECAR_SUFFIXES, q_ads, purge_assets, _finish_purge,
    _restore_staged_media,
)
from peach.web_contract import WebContract

FIELDS = ["id", "score", "location", "medium", "size_mb", "name", "why", "path"]
TRASHABLE_LOCATIONS = SCANNABLE_LOCATIONS


def load_review(path: Path) -> list[dict]:
    """只接受逐文件确认的删除清单,评分不构成永久删除依据。"""
    rows = read_rows(path)
    selected = [row for row in rows if row.get("decision") == "delete"]
    seen = set()
    for row in selected:
        asset_id = int(row["id"])
        if asset_id in seen or not row.get("path") or not row.get("why"):
            raise ValueError("删除清单缺少路径、证据或包含重复资产")
        confidence = float(row.get("confidence") or 0)
        if not math.isfinite(confidence) or not .95 <= confidence <= 1:
            raise ValueError("删除清单的确认置信度不足 0.95")
        if int(row["size"]) < 0:
            raise ValueError("删除清单的文件体积不能为负")
        seen.add(asset_id)
    return selected


def candidate_ids(db_path: Path) -> set[int]:
    """当前仍在垃圾复核队列(未被标「不是垃圾」)里的资产 id,只读。"""
    result = q_ads(WebContract(Path(db_path)), limit=1_000_000)
    return {int(item["id"]) for item in result["items"]}


def sidecar_files(path: Path) -> list[str]:
    """盘上与该文件同名的字幕与 NFO(`片名.srt`、`片名.zh.ass`、`片名.nfo` 等)。"""
    prefix = path.stem.casefold() + "."
    return sorted(entry.name for entry in path.parent.iterdir()
                  if entry != path and entry.name.casefold().startswith(prefix)
                  and entry.suffix.casefold() in MEDIA_SIDECAR_SUFFIXES)


def skip_reason(connection, row, path: Path, candidates: set[int], holders: set[int]) -> str:
    """不能永久删除的原因;空串表示可以删。"""
    if connection.execute(
            "SELECT 1 FROM review_decision WHERE category='junk_file' AND status='rejected' "
            "AND item_key=?", (str(row["id"]),)).fetchone():
        return "用户已标为不是垃圾"
    if int(row["id"]) not in candidates:
        return "已不在垃圾复核队列里"
    if int(row["id"]) in holders:
        return "带个人记录"
    if connection.execute("SELECT 1 FROM asset_subtitle WHERE asset_id=?", (row["id"],)).fetchone():
        return "登记了字幕"
    sidecars = sidecar_files(path)
    if sidecars:
        return "盘上有同名字幕或资料:" + "、".join(sidecars)
    return ""


def purge_reviewed(connection, reviews: list[dict], candidates: set[int]) -> dict:
    """核对清单与文件后,复用批量永久删除的隔离、提交与清退流程。

    `candidates` 是删除前只读算出的垃圾复核队列 id;不在其中的行跳过。
    """
    baseline = {tuple(row) for row in connection.execute("PRAGMA foreign_key_check")}
    outcome = None
    skipped = []
    try:
        connection.execute("BEGIN IMMEDIATE")
        holders = record_holders(connection, [int(review["id"]) for review in reviews])
        rows = []
        for review in reviews:
            row = connection.execute("SELECT * FROM asset WHERE id=?", (int(review["id"]),)).fetchone()
            if row is None or row["location"] not in TRASHABLE_LOCATIONS or row["path"] != review["path"]:
                raise ValueError("删除清单已失效:资产来源或路径不一致")
            path = translate_ledger_path(row["path"])
            roots = [translate_ledger_path(root) for root in LOCATION_ROOT_DECLARATIONS.get(row["location"], ())]
            if not any(root_online(root) and within_root(path, root) for root in roots):
                raise ValueError("删除路径不在已挂载的来源根内")
            if path.suffix.casefold() in MEDIA_SIDECAR_SUFFIXES:
                raise ValueError("媒体资料与字幕不能进入垃圾永久删除")
            if path.is_symlink() or not path.is_file() or path.stat().st_size != int(review["size"]):
                raise ValueError("删除清单已失效:文件不存在、类型或体积变化")
            reason = skip_reason(connection, row, path, candidates, holders)
            if reason:
                skipped.append({"id": row["id"], "path": row["path"], "reason": reason})
                continue
            rows.append(row)
        outcome = purge_assets(connection, rows)
        integrity, _ = verify_after_write(connection)
        violations = {tuple(row) for row in connection.execute("PRAGMA foreign_key_check")}
        if integrity != "ok" or violations - baseline:
            raise RuntimeError("删除后账本校验失败")
        connection.commit()
    except BaseException:
        connection.rollback()
        if outcome is not None:
            _restore_staged_media(outcome["_staged"])
        raise
    return {**_finish_purge(outcome), "skipped": skipped, "existing_foreign_keys": len(baseline)}


def select_candidates(db_path: Path, *, min_score: int, locations: tuple[str, ...] = (),
                      kind: str = "") -> list[dict]:
    """只读跑一遍垃圾判据,按分数与来源过滤,并补上路径供人核对。"""
    contract = WebContract(Path(db_path))
    result = q_ads(contract, limit=100000, kind=kind)
    items = [item for item in result["items"] if int(item["score"]) >= min_score]
    if locations:
        items = [item for item in items if item["location"] in locations]
    connection = open_readonly(db_path)
    try:
        paths = {row["id"]: row["path"] for row in connection.execute("SELECT id,path FROM asset")}
    finally:
        connection.close()
    for item in items:
        item["path"] = paths.get(item["id"], "")
    return items


def trash_assets(connection, asset_ids: list[int]) -> int:
    """与复核页「移入回收站」同义:只改 `disposal`,不动文件与引用关系。"""
    ids = list(dict.fromkeys(int(value) for value in asset_ids))
    if not ids:
        return 0
    marks = ",".join("?" * len(ids))
    sources = ",".join("?" * len(TRASHABLE_LOCATIONS))
    cursor = connection.execute(
        f"UPDATE asset SET disposal='trash', disposal_reason='junk', feedback_at=? WHERE id IN ({marks}) "
        f"AND disposal IS NULL AND location IN ({sources})",
        [time.time(), *ids, *TRASHABLE_LOCATIONS],
    )
    connection.commit()
    return cursor.rowcount


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="把高置信广告残留移入回收站")
    add_ledger_write_args(parser, db_default=DATABASE_PATH)
    parser.add_argument("--min-score", type=int, default=AUTO_TRASH_SCORE,
                        help=f"垃圾评分下限;默认 {AUTO_TRASH_SCORE}(整个名字都是推广语一档,"
                             "固定命名与未探测时长视频的目录证据单独到不了)")
    parser.add_argument("--location", action="append", choices=TRASHABLE_LOCATIONS)
    parser.add_argument("--kind", choices=("video", "image", "audio", "archive", "url", "other"),
                        default="")
    parser.add_argument("--out", type=Path)
    parser.add_argument("--review-csv", type=Path, help="含 id、path、size、decision、confidence、why 的逐文件复核清单")
    parser.add_argument("--purge", action="store_true", help="永久删除清单中确认的垃圾,必须给 --review-csv;默认仍只出计划")
    return parser


def main(argv: list[str] | None = None) -> int:
    args = build_parser().parse_args(argv)
    if args.purge:
        if not args.review_csv:
            raise SystemExit("--purge 必须同时给 --review-csv")
        reviewed = load_review(args.review_csv)
        if not args.apply:
            print(f"清单确认删除 {len(reviewed)} 条;未加 --apply,只核对清单。")
            return 0
        candidates = candidate_ids(args.db)
        connection = open_for_write(args)
        try:
            result = purge_reviewed(connection, reviewed, candidates)
        finally:
            connection.close()
        report = args.out or args.review_csv.with_suffix(".result.json")
        report.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
        print(json.dumps(result, ensure_ascii=False))
        return 1 if result["blocked"] or result["cleanup_pending"] or result["skipped"] else 0
    locations = tuple(args.location or ())
    selected = select_candidates(args.db, min_score=args.min_score,
                                 locations=locations, kind=args.kind)
    output = args.out or GENERATED_DIR / time.strftime("junk-trash-%Y%m%d-%H%M%S.csv")
    rows = [{field: item.get(field, "") for field in FIELDS} for item in selected]
    for row, item in zip(rows, selected):
        row["size_mb"] = round((item.get("size") or 0) / 1048576, 1)
    write_rows(output, FIELDS, rows, atomic=True)

    total_gb = sum((item.get("size") or 0) for item in selected) / 1024 ** 3
    counts: dict[str, int] = {}
    for item in selected:
        counts[item["location"]] = counts.get(item["location"], 0) + 1
    print(f"评分≥{args.min_score} 的候选 {len(selected)} 条 / {total_gb:.2f} GB,"
          f"按来源 {counts or '无'} → {output}")
    for item in selected[:15]:
        print(f"  {item['score']:>3} {item['location']:<6} {item['medium']:<6} "
              f"{(item.get('size') or 0)/1048576:>9.1f}MB {str(item.get('name'))[:44]!r}"
              f" | {item.get('why')}")
    if len(selected) > 15:
        print(f"  …… 其余 {len(selected) - 15} 条见 CSV")

    if not args.apply:
        print("未加 --apply,只出清单;文件仍在原处。")
        return 0

    connection = open_for_write(args)
    try:
        before = counts_of(connection, {
            "trash": "SELECT count(*) FROM asset WHERE disposal='trash'",
        })
        changed = trash_assets(connection, [int(item["id"]) for item in selected])
        after = counts_of(connection, {
            "trash": "SELECT count(*) FROM asset WHERE disposal='trash'",
        })
        integrity, foreign_keys = verify_after_write(connection)
    finally:
        connection.close()
    if integrity != "ok" or foreign_keys:
        raise RuntimeError(f"写入后 ledger 校验失败:integrity={integrity} foreign_keys={foreign_keys}")
    print(f"已移入回收站 {changed} 条;回收站 {before['trash']} → {after['trash']}。"
          f"备份 {args.backup};文件未删除,清空回收站才会真正删除。")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())