Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

6.8 KB
Raw Blame
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""115 SHA1 对账:读取服务端文件清单,按文件名和大小写入 ledger。

安装 maintenance-115 可选依赖后运行;首次登录使用 115 手机 App 扫码。
Cookie 默认存入配置 Peach 数据目录的 secrets/115-cookies.txt,重复报告存入 review/。
--cookie-file / PEACH_115_COOKIE_FILE 可指定已有 Cookie 文件;
--dupes-out / PEACH_115_DUPES_OUT 可指定重复报告位置,需同时启用 --dupes。
重复项默认按目录深度与 Provider 自动目录排序;--deprioritize-dir 可重复指定
优先删除的目录名,只生成复核报告,不删除文件。
"""
import argparse
import csv
import os
import sqlite3
import sys
import time
from collections import defaultdict
from contextlib import closing
from pathlib import Path, PureWindowsPath

PROJECT_ROOT = Path(__file__).resolve().parent.parent
SRC_DIR = PROJECT_ROOT / "src"
if str(SRC_DIR) not in sys.path:
    sys.path.insert(0, str(SRC_DIR))

from peach.review_csv import ENCODING
from peach.config import DATABASE_PATH, LOG_DIR, REVIEW_DIR, SECRETS_DIR


def parse_args(argv=None):
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--cookie-file", type=Path,
                        default=Path(os.environ.get("PEACH_115_COOKIE_FILE")
                                     or SECRETS_DIR / "115-cookies.txt"),
                        help="115 Cookie 文件,默认使用配置 Peach 数据目录的 secrets 目录")
    parser.add_argument("--dupes", action="store_true", help="写入后生成 SHA1 重复报告")
    parser.add_argument("--dupes-out", type=Path,
                        default=Path(os.environ.get("PEACH_115_DUPES_OUT")
                                     or REVIEW_DIR / "115-sha1重复.csv"),
                        help="重复报告路径,默认使用配置 Peach 数据目录的 review 目录")
    parser.add_argument("--deprioritize-dir", action="append", default=[],
                        help="重复报告中优先删除的目录名,可重复指定")
    args = parser.parse_args(argv)
    args.cookie_file = args.cookie_file.expanduser()
    args.dupes_out = args.dupes_out.expanduser()
    return args


def duplicate_score(path, deprioritize_dirs=()):
    parts = PureWindowsPath(path).parts
    lowered = {part.casefold() for part in parts[:-1]}
    score = -len(parts[:-1]) * 2
    if lowered.intersection(name.casefold() for name in deprioritize_dirs):
        score -= 50
    if "auto_create@" in path.lower():
        score -= 40
    return score


def write_duplicate_report(conn, output, deprioritize_dirs=()):
    groups = defaultdict(list)
    for aid, path, size, sha in conn.execute(
            "SELECT id,path,size,hash FROM asset WHERE location='115' AND hash IS NOT NULL"):
        groups[sha].append((aid, path, size))
    duplicates = {sha: items for sha, items in groups.items() if len(items) > 1}
    waste = sum(items[0][2] * (len(items) - 1) for items in duplicates.values())
    output.parent.mkdir(parents=True, exist_ok=True)
    with output.open("w", newline="", encoding=ENCODING) as stream:
        writer = csv.writer(stream)
        writer.writerow(["组号", "动作", "大小GB", "路径", "SHA1"])
        ordered = sorted(duplicates.items(), key=lambda item: -item[1][0][2] * (len(item[1]) - 1))
        for index, (sha, items) in enumerate(ordered, 1):
            items = sorted(items, key=lambda item: -duplicate_score(item[1], deprioritize_dirs))
            for position, (_aid, path, size) in enumerate(items):
                writer.writerow([index, "保留" if position == 0 else "删除",
                                 round(size / 1024**3, 3), path, sha])
    return len(duplicates), waste


def main(argv=None):
    args = parse_args(argv)
    try:
        from p115client import P115Client
        from p115client.tool.iterdir import iter_files
    except ImportError:
        print('缺少 maintenance-115 可选依赖,请安装后运行。', file=sys.stderr)
        return 1

    if not DATABASE_PATH.is_file():
        print(f"账本不存在:{DATABASE_PATH}", file=sys.stderr)
        return 1
    LOG_DIR.mkdir(parents=True, exist_ok=True)
    log_path = LOG_DIR / "sha1-{}.log".format(time.strftime("%Y%m%d-%H%M%S"))
    with log_path.open("w", encoding="utf-8", buffering=1) as log_stream:
        def log(message):
            print(message, flush=True)
            log_stream.write(message + "\n")

        if args.cookie_file.is_file():
            client = P115Client(str(args.cookie_file))
            log(f"用已保存的 Cookie 登录:{args.cookie_file}")
        else:
            log("首次运行,需要扫码登录(用 115 手机 App 扫终端里的二维码)")
            client = P115Client(console_qrcode=True)
            args.cookie_file.parent.mkdir(parents=True, exist_ok=True)
            args.cookie_file.write_text(client.cookies_str, encoding="utf-8")
            log(f"Cookie 已保存到 {args.cookie_file}(仅本机,勿外传)")

        log("开始拉取全量文件清单(只读元数据,不下载任何文件内容)…")
        rows = []
        started = time.time()
        for index, item in enumerate(iter_files(client, 0, type=99, with_path=True), 1):
            sha = item.get("sha1") or item.get("sha")
            if not sha:
                continue
            rows.append((sha, item.get("name") or "", int(item.get("size") or 0)))
            if index % 5000 == 0:
                log(f"  {index:,} 条  {time.time()-started:.0f}s")
        log(f"共 {len(rows):,} 条带 SHA1 的文件,耗时 {time.time()-started:.0f}s")

        # 云端与 ledger 的路径口径不同,使用文件名和大小匹配。
        with closing(sqlite3.connect(str(DATABASE_PATH), timeout=120)) as conn:
            have = defaultdict(list)
            for aid, name, size in conn.execute("SELECT id,name,size FROM asset WHERE location='115'"):
                have[(name, size)].append(aid)
            log(f"账本里 115 的记录 {sum(len(ids) for ids in have.values()):,} 条")
            updates, missed = [], 0
            for sha, name, size in rows:
                ids = have.get((name, size))
                if ids:
                    updates.extend(("sha1", sha, aid) for aid in ids)
                else:
                    missed += 1
            conn.executemany("UPDATE asset SET hash_kind=?, hash=? WHERE id=?", updates)
            conn.commit()
            log(f"已写入 SHA1 {len(updates):,} 条;云端有但账本没匹配上的 {missed:,} 条")
            if args.dupes:
                count, waste = write_duplicate_report(conn, args.dupes_out, args.deprioritize_dir)
                log(f"SHA1 精确重复:{count:,} 组,可回收 {waste/1024**3:.1f} GB")
                log(f"→ {args.dupes_out}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())