Skip to content

armgit / peach

Public Imported from GitHub
0

Peach: Local-first video metadata scraper and media manager.

11.8 KB
Raw Blame
r"""把头像上的站点水印裁掉或修掉,原图归档。

用法(默认只看不写):

    python scripts/scrub_avatar_watermarks.py
    python scripts/scrub_avatar_watermarks.py --apply
    python scripts/scrub_avatar_watermarks.py --marks my-boxes.csv --apply

**这一步是半自动的,别指望跑一遍就干净。** 检出走 PP-OCRv3 的 DB 文字检测头,
实心水印(`PRIVATE.com`、`TEAMSKEET.COM`、角落的 `DogFart (SM)`)一抓就中;
叠在皮肤上的半透明水印(`NUBILES.NET`、`MATTIEDOLL`)它给不出框,判据和局限写在
`peach.avatar_watermark` 的模块说明里。所以默认这一趟只写两样东西:一份候选 CSV,
和一叠画了框的标注图。人看图确认,检出漏掉的自己往 CSV 里补一行框,再带 `--marks`
跑 `--apply`。

`--apply` 做的事:原图连同 `.ct`、`.provenance.json`、`.face.json` 三个边车整套
搬进 `avatars-superseded/`,处理后的图写回原路径,provenance 补一段
`watermark_scrubbed` 记下裁了哪里、修了几处和新的哈希,取景 sidecar 按新图重算。
留原图是因为这一步不可逆:裁切改了构图,JPEG 重编码改了每一个像素。

JPEG 重编码这件事值得说明:裁切本身能做到无损,但那要 `jpegtran` 级别的工具链,
而这批图的用途是页面上一张 320px 的头像。这里按质量 95 重编码,换来的是不引入
新的外部依赖。
"""
from __future__ import annotations

import argparse
import csv
import hashlib
import json
import os
import sys
import time
import uuid
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

from peach import avatar_face, avatar_watermark, face_detect  # noqa: E402
from peach.config import GENERATED_DIR  # noqa: E402

#: 处理记录写在 provenance 的这个键下。有这个键就是已经处理过,不再重复处理。
STAMP = "watermark_scrubbed"
#: mime → OpenCV 编码器认的扩展名。`.img` 文件本身不带格式,格式在 `.ct` 边车里。
ENCODERS = {"image/jpeg": ".jpg", "image/png": ".png", "image/webp": ".webp",
            "image/bmp": ".bmp"}
JPEG_QUALITY = 95
CSV_FIELDS = ("file", "x", "y", "w", "h", "score", "origin", "plan", "note")


def sidecars(image: Path) -> tuple[Path, Path, Path]:
    return (image.with_name(f"{image.name}.ct"),
            image.with_name(f"{image.name}.provenance.json"),
            avatar_face.sidecar_path(image))


def provenance(image: Path) -> dict:
    _, path, _ = sidecars(image)
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except (OSError, ValueError):
        return {}


def mime_of(image: Path) -> str:
    ct, _, _ = sidecars(image)
    try:
        return ct.read_text(encoding="utf-8").strip()
    except OSError:
        return ""


def face_box(detector, image):
    """主角那张脸在原图里的像素框,用来卡住裁切线。检不出脸就不设这项约束。"""
    faces = detector.detect(image)
    if not faces:
        return None
    best = face_detect.main_face(faces)
    height, width = image.shape[:2]
    return (int(best.cx * width - best.width * width / 2),
            int(best.cy * height - best.height * height / 2),
            int(best.width * width), int(best.height * height))


def read_marks(path: Path | None) -> dict[str, list[avatar_watermark.Mark]]:
    """人工确认或补写的框。只读 file 与四个坐标,其余列随便写什么都不影响。"""
    if path is None:
        return {}
    out: dict[str, list[avatar_watermark.Mark]] = {}
    with path.open("r", encoding="utf-8", newline="") as handle:
        for row in csv.DictReader(handle):
            name = (row.get("file") or "").strip()
            if not name:
                continue
            try:
                box = [int(float(row[key])) for key in ("x", "y", "w", "h")]
            except (KeyError, TypeError, ValueError):
                continue
            if box[2] > 0 and box[3] > 0:
                out.setdefault(name, []).append(avatar_watermark.Mark(*box))
    return out


def encode(image, mime: str):
    """按原格式重新编码。认不出格式的不处理——猜一个格式等于悄悄换掉文件类型。"""
    import cv2

    suffix = ENCODERS.get(mime)
    if suffix is None:
        return None
    params = [cv2.IMWRITE_JPEG_QUALITY, JPEG_QUALITY] if suffix == ".jpg" else []
    ok, buffer = cv2.imencode(suffix, image, params)
    return buffer.tobytes() if ok else None


def archive(image: Path, target: Path) -> None:
    """原图连边车整套搬走。搬不是复制:原路径马上要被处理后的图占用。"""
    target.mkdir(parents=True, exist_ok=True)
    for source in (image, *sidecars(image)):
        if source.exists():
            os.replace(source, target / source.name)


def install(image: Path, data: bytes, record: dict, info: dict,
            detector) -> None:
    """处理后的图落回原路径,provenance 补一段处理记录,取景 sidecar 按新图重算。"""
    staging = image.with_name(f"{image.name}.{uuid.uuid4().hex}.tmp")
    staging.write_bytes(data)
    os.replace(staging, image)
    width, height = info["px"]
    record[STAMP] = {
        "at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
        "marks": info["marks"],
        "crop": info["crop"],
        "inpainted": info["inpainted"],
        "model": avatar_watermark.MODEL_PATH.name,
    }
    record["sha256"] = hashlib.sha256(data).hexdigest()
    record["width"], record["height"] = width, height
    _, provenance_path, face_path = sidecars(image)
    provenance_path.write_text(
        json.dumps(record, ensure_ascii=False, indent=2), encoding="utf-8")
    # 旧 sidecar 记的是裁切前的坐标。留着它,页面会拿旧框给新图取景,放大到一个
    # 空位置上——而那在界面上和「这张图本来就该这么显示」看不出区别。
    face_path.unlink(missing_ok=True)
    fresh = avatar_face.face_record(image, detector)
    if fresh:
        avatar_face.write_sidecar(image, fresh)


def annotate(image, marks, result, target: Path) -> None:
    """左右对照图:左边画着框的原图,右边处理后的结果。人就看这一张判断。"""
    import cv2
    import numpy as np

    marked = image.copy()
    for mark in marks:
        cv2.rectangle(marked, (mark.x, mark.y),
                      (mark.x + mark.w, mark.y + mark.h), (0, 0, 255), 3)
        cv2.putText(marked, f"{mark.score:.2f}", (mark.x, max(14, mark.y - 6)),
                    cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
    panels = []
    for panel in (marked, result):
        scale = 380 / panel.shape[1]
        panels.append(cv2.resize(panel, (380, int(panel.shape[0] * scale))))
    tall = max(panel.shape[0] for panel in panels)
    pair = np.hstack([np.pad(panel, ((0, tall - panel.shape[0]), (0, 0), (0, 0)))
                      for panel in panels])
    target.parent.mkdir(parents=True, exist_ok=True)
    cv2.imwrite(str(target), pair)


def plan_of(info: dict) -> str:
    if info["crop"] and info["inpainted"]:
        return "crop+inpaint"
    if info["crop"]:
        return "crop"
    if info["inpainted"]:
        return "inpaint"
    return "none"


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(description="头像去水印")
    parser.add_argument("--source", type=Path, default=GENERATED_DIR / "avatars")
    parser.add_argument("--archive", type=Path,
                        default=GENERATED_DIR / "avatars-superseded")
    parser.add_argument("--review", type=Path,
                        default=GENERATED_DIR / "watermark-review")
    parser.add_argument("--candidates", type=Path,
                        default=GENERATED_DIR / "watermark-candidates.csv")
    parser.add_argument("--marks", type=Path,
                        help="人工确认或补写的框,CSV,列 file,x,y,w,h")
    parser.add_argument("--model", type=Path, help="文字检测模型,默认按固定 URL 取")
    parser.add_argument("--limit", type=int, default=0)
    parser.add_argument("--force", action="store_true",
                        help="已经处理过的也重新处理")
    parser.add_argument("--apply", action="store_true", help="真的写盘")
    return parser


def main(argv: list[str] | None = None) -> int:
    import cv2

    args = build_parser().parse_args(argv)
    manual = read_marks(args.marks)
    detector = avatar_watermark.MarkDetector(args.model)
    faces = face_detect.FaceDetector()
    rows: list[dict] = []
    summary = {"ok": True, "seen": 0, "with_marks": 0, "content": 0,
               "written": 0, "skipped": 0, "reasons": []}
    reasons: set[str] = set()
    for image_path in sorted(args.source.glob("*.img")):
        if args.limit and summary["seen"] >= args.limit:
            break
        summary["seen"] += 1
        record = provenance(image_path)
        if STAMP in record and not args.force:
            summary["skipped"] += 1
            reasons.add("already_scrubbed")
            continue
        image = cv2.imread(str(image_path))
        if image is None:
            summary["skipped"] += 1
            reasons.add("unreadable")
            continue
        height, width = image.shape[:2]
        found = [mark for mark in detector.detect(image)
                 if avatar_watermark.on_edge(mark, width, height)]
        hand = manual.get(image_path.name, [])
        marks = found + hand
        if not marks:
            continue
        summary["with_marks"] += 1
        # 「这是画面文字不是水印」只用来质疑检出器。人手写的框是已经看过图的判断,
        # 不该被同一条数量判据推翻——补框的场合本来就是检出器判错了。
        content = not hand and avatar_watermark.looks_like_content(found)
        if content:
            summary["content"] += 1
            reasons.add("looks_like_content")
            result, info = image, {"crop": None, "inpainted": 0, "marks": len(marks),
                                   "px": [width, height]}
        else:
            face = face_box(faces, image)
            result, info = avatar_watermark.scrub(image, marks, face)
        plan = "content" if content else plan_of(info)
        # 来源按框是从哪一边来的记,不从分数反推:检出的分数也可以正好是 1.0。
        tagged = [(mark, "detected") for mark in found]
        tagged += [(mark, "manual") for mark in hand]
        for mark, origin in tagged:
            rows.append({
                "file": image_path.name, "x": mark.x, "y": mark.y,
                "w": mark.w, "h": mark.h, "score": mark.score,
                "origin": origin, "plan": plan,
                "note": f"{width}x{height}",
            })
        annotate(image, marks, result, args.review / f"{image_path.stem}.png")
        if content or not args.apply:
            continue
        mime = mime_of(image_path)
        data = encode(result, mime)
        if data is None:
            summary["skipped"] += 1
            reasons.add("unsupported_format")
            continue
        archive(image_path, args.archive)
        # 格式没变,所以 `.ct` 的内容照抄;它刚跟着原图被搬进归档目录,得补回来。
        image_path.with_name(f"{image_path.name}.ct").write_text(
            mime, encoding="utf-8")
        install(image_path, data, record, info, faces)
        summary["written"] += 1
    args.candidates.parent.mkdir(parents=True, exist_ok=True)
    with args.candidates.open("w", encoding="utf-8", newline="") as handle:
        writer = csv.DictWriter(handle, fieldnames=CSV_FIELDS)
        writer.writeheader()
        writer.writerows(rows)
    summary["reasons"] = sorted(reasons)
    summary["candidates"] = str(args.candidates)
    summary["review"] = str(args.review)
    print(json.dumps(summary, ensure_ascii=False))
    return 0


if __name__ == "__main__":
    sys.exit(main())