Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
from __future__ import annotations
import sqlite3
from collections import Counter
from pathlib import Path
from typing import TYPE_CHECKING, Literal
import pytest
from cyberdrop_dl.hasher import Hasher, _compute_hash, hash_directory
if TYPE_CHECKING:
from cyberdrop_dl.manager import Manager
def get_hashes(path: Path) -> set[tuple[str, str]]:
query = "SELECT hash_type, hash FROM hash "
conn = sqlite3.connect(path)
try:
cursor = conn.execute(query)
return set(cursor.fetchall())
finally:
conn.close()
def create_files(path: Path, number: int) -> None:
for i in range(number):
file = path / f"file_{i}"
file.write_text(str(i), encoding="utf-8")
@pytest.mark.parametrize(
"expected_results",
[
{
("xxh128", "cb358fcee0dfde56fb95a7322f5da314"),
("xxh128", "df3ce784d856334d65cd25028f98f158"),
("xxh128", "ce8f15881282a4001982e3a7bb241055"),
},
{
("xxh128", "ce8f15881282a4001982e3a7bb241055"),
("md5", "cfcd208495d565ef66e7dff9f98764da"),
},
{
("xxh128", "ce8f15881282a4001982e3a7bb241055"),
("md5", "cfcd208495d565ef66e7dff9f98764da"),
("sha256", "5feceb66ffc86f38d952786c6d696c79c2dbc239dd4e91b46729d73a27fb57e9"),
},
],
)
async def test_hash_directory_scanner(manager: Manager, expected_results: set[tuple[str, str]]) -> None:
count = Counter(x[0] for x in expected_results)
n_files = max(count.values())
algos = count.keys()
assert len(expected_results) == len(algos) * n_files
options = manager.config.hashing
options.algorithms = tuple(algos) # pyright: ignore[reportAttributeAccessIssue]
options.re_compute()
manager.config.download_folder.mkdir(parents=True)
db_path = manager.appdata.db_file
with Hasher.create(manager.config, manager.database) as hasher:
await hash_directory(hasher)
assert not get_hashes(db_path)
create_files(manager.config.download_folder, n_files)
with Hasher.create(manager.config, manager.database) as hasher:
await hash_directory(hasher)
results = get_hashes(db_path)
assert len(results) == len(expected_results)
assert results == expected_results
async def test_hash_directory_does_not_crash_with_subfolders(tmp_cwd: Path, manager: Manager) -> None:
options = manager.config.hashing
options.algorithms = "md5", "sha256"
options.re_compute()
hash_folder = tmp_cwd / "sorted_downloads"
here = Path(__file__).parent
files = [hash_folder / f.relative_to(here) for f in here.rglob("*") if f.is_file()]
assert len(files) >= 10
for file in files:
file.parent.mkdir(parents=True, exist_ok=True)
file.touch()
with Hasher.create(manager.config, manager.database, hash_folder) as hasher:
await hash_directory(hasher)
@pytest.mark.parametrize(
("algo", "expected"),
[
("xxh128", "ae6ea5d955361e9dd7d91f1432616dcc"),
("md5", "1ebbd3e34237af26da5dc08a4e440464"),
("sha256", "3972dc9744f6499f0f9b2dbf76696f2ae7ad8af9b23dde66d6af86c9dfb36986"),
],
)
def test_compute_hash(tmp_cwd: Path, algo: Literal["xxh128", "md5", "sha256"], expected: str) -> None:
license_file = Path(__file__).parent.parent / "LICENSE"
file = tmp_cwd / "license"
file.write_bytes(license_file.read_text(encoding="utf8").encode()) # Remove windows EOL
result = _compute_hash(file, algo)
assert result == expected