Skip to content

Bulk asynchronous downloader for multiple file hosts

7.2 KB
Raw Blame
import datetime as dt
import itertools
import shutil
from pathlib import Path
from typing import Never

import pytest

from cyberdrop_dl.sorter import Sorter, _format_dest, _have_same_content, _move_file

DOWNLOADS = Path("/mnt/home/user/downloads/cdl/")
SORT_DIR = DOWNLOADS.parent / "cdl_sorted"
MTIME = dt.datetime(2023, 7, 14, 12, 34, 56, tzinfo=dt.UTC).timestamp()


@pytest.mark.parametrize(
    ("base_dir", "format_str", "expected"),
    [
        (
            "album 1 (site)",
            ("{sort_dir}/{base_dir}/audios/{parent_dir}___{filename}{ext}"),
            "/mnt/home/user/downloads/cdl_sorted/album 1 (site)/audios/sub album___song.mp3",
        ),
        ("foo (Mega.NZ)", "{base_dir}/{file_date_iso}/{filename}{ext}", "foo (Mega.NZ)/2023-07-14/song.mp3"),
        (
            "folder 1",
            "/mnt/data/{base_dir}/{file_date_us}/{filename}{ext}",
            "/mnt/data/folder 1/2023-14-07/song.mp3",
        ),
        ("folder 1", "{base_dir}/{file_date:%Y}/{filename}{ext}", "folder 1/2023/song.mp3"),
        ("folder 1", "{base_dir}/{invalid_param}/{filename}{ext}", "folder 1/UNKNOWN_INVALID_PARAM/song.mp3"),
    ],
)
def test_destination_format(base_dir: str, format_str: str, expected: str) -> None:
    file = DOWNLOADS / base_dir / "sub album/song.mp3"
    result = _format_dest(file, base_dir, format_str, MTIME, SORT_DIR)
    assert result == Path(expected)


class TestHaveSameContent:
    def test_same_file_returns_true(self, tmp_path: Path) -> None:
        foo = tmp_path / "foo.bin"
        foo.write_bytes(b"12345")
        assert _have_same_content(foo, foo)

    def test_same_content_same_size_returns_true(self, tmp_path: Path) -> None:
        foo, bar = tmp_path / "foo.bin", tmp_path / "bar.bin"
        data = b"0" * 1_000
        foo.write_bytes(data)
        bar.write_bytes(data)
        assert _have_same_content(foo, bar)

    def test_different_size_returns_false(self, tmp_path: Path) -> None:
        foo, bar = tmp_path / "foo.bin", tmp_path / "bar.bin"
        foo.write_bytes(b"x")
        bar.write_bytes(b"x" * 10)
        assert not _have_same_content(foo, bar)

    def test_same_size_different_content_returns_false(self, tmp_path: Path) -> None:
        foo, bar = tmp_path / "foo.bin", tmp_path / "bar.bin"
        foo.write_bytes(b"\x00" * 512)
        bar.write_bytes(b"\xff" * 512)
        assert not _have_same_content(foo, bar)

    def test_empty_files_return_true(self, tmp_path: Path) -> None:
        foo, bar = tmp_path / "foo.bin", tmp_path / "bar.bin"
        foo.touch()
        bar.touch()
        assert _have_same_content(foo, bar)

    def test_large_files_same_content_return_true(self, tmp_path: Path) -> None:
        foo, bar = tmp_path / "foo.bin", tmp_path / "bar.bin"
        data_10MB = b"A" * 1024 * 1024 * 10
        foo.write_bytes(data_10MB)
        bar.write_bytes(data_10MB)
        assert _have_same_content(foo, bar)


class TestMoveFile:
    def test_same_file_returns_immediately(self, tmp_path: Path) -> None:
        foo = tmp_path / "foo.txt"
        foo.write_text("x", encoding="utf8")
        assert _move_file(foo, foo) == foo
        assert foo.exists()

    def test_simple_move(self, tmp_path: Path) -> None:
        src = tmp_path / "foo.txt"
        src.write_text("data", encoding="utf8")
        dst = tmp_path / "sub" / "bar.txt"
        out = _move_file(src, dst)
        assert out == dst
        assert not src.exists()
        assert dst.read_text(encoding="utf8") == "data"

    def test_name_collision_same_size_diferent_content(self, tmp_path: Path) -> None:
        src = tmp_path / "foo.txt"
        src.write_text("abc", encoding="utf8")
        dst = tmp_path / "bar.txt"
        dst.write_text("123", encoding="utf8")
        out = _move_file(src, dst)

        assert out
        assert out != dst
        assert not src.exists()
        assert dst.read_text(encoding="utf8") == "123"
        assert out.read_text(encoding="utf8") == "abc"

    def test_name_collision_different_size(self, tmp_path: Path) -> None:
        src = tmp_path / "foo.jpg"
        src.write_bytes(b"0" * 100)
        dst = tmp_path / "bar.jpg"
        dst.write_bytes(b"1" * 50)
        out = _move_file(src, dst)
        assert out
        assert out == tmp_path / "bar1.jpg"
        assert not src.exists()
        assert out.stat().st_size == 100

    def test_custom_incrementer_format(self, tmp_path: Path) -> None:
        src = tmp_path / "foo.log"
        src.write_text("a", encoding="utf8")
        dest = tmp_path / "bar.log"
        dest.write_text("x", encoding="utf8")
        for i in range(1, 4):
            (tmp_path / f"bar_{i}.log").write_text("x", encoding="utf8")
        out = _move_file(src, dest, incrementer_format="_{i}")
        assert out == tmp_path / "bar_4.log"

    def test_max_retries_exceeded(self, tmp_path: Path) -> None:
        src = tmp_path / "foo.txt"
        src.write_text("x", encoding="utf8")
        dest = tmp_path / "bar.txt"
        dest.touch()
        for i in range(1, 12):
            (tmp_path / f"bar{i}.txt").touch()

        assert _move_file(src, dest) is None
        assert src.exists()

    def test_os_error_during_move_cancels_it(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
        src = tmp_path / "foo.txt"
        src.write_text("x", encoding="utf8")
        dst = tmp_path / "bar.txt"

        def boom(*_: object, **_k: object) -> Never:
            raise OSError

        monkeypatch.setattr(shutil, "move", boom)
        assert _move_file(src, dst) is None
        assert src.exists()


class Files:
    VIDEOS = "video.mp4", "movies/4K/a_movie.mkv"
    OTHERS = (
        "docker-compose.yml",
        "index.html",
        "styles.css",
        "tests/scripts/notes.txt",
        "tests/scripts/script.js",
        "tests/setup.sh",
        "tests/utils.py",
    )
    AUDIOS = ("music/song.mp3",)
    IMAGES = ("logo.png", "tests/logo.jpg")
    TEMP = ("tmp/download.part",)

    ALL = *VIDEOS, *OTHERS, *AUDIOS, *IMAGES, *TEMP


async def test_sorter(tmp_path: Path) -> None:
    assert len(Files.ALL) == len(set(Files.ALL))
    input_dir = tmp_path / "cdl_downloads"
    output_dir = tmp_path / "cdl_sorted_downloads"
    sorter = Sorter(
        input_dir=input_dir,
        output_dir=output_dir,
        audio_format="{sort_dir}/audio/{filename}{ext}",
        image_format="{sort_dir}/image/{filename}{ext}",
        video_format="{sort_dir}/video/{filename}{ext}",
        non_media_format="{sort_dir}/other/{filename}{ext}",
    )

    for idx, file in enumerate(Files.ALL):
        path = input_dir / file
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(str(idx), encoding="utf8")

    await sorter.run(disable_tui=True)
    stats = sorter.stats
    assert stats.total == len(Files.ALL) - len(Files.TEMP)
    assert stats.videos == len(Files.VIDEOS)
    assert stats.others == len(Files.OTHERS)

    assert sorted(f for f in output_dir.rglob("*") if f.is_file()) == sorted(
        itertools.chain.from_iterable(
            (output_dir / kind / file.name for file in map(Path, files))
            for kind, files in [
                ("audio", Files.AUDIOS),
                ("image", Files.IMAGES),
                ("video", Files.VIDEOS),
                ("other", Files.OTHERS),
            ]
        )
    )