Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
from __future__ import annotations
from pathlib import Path
import pytest
from cyberdrop_dl import aio, scrape_mapper
from cyberdrop_dl.config import Config
from cyberdrop_dl.constants import USE_RETRY_PATH
from cyberdrop_dl.crawlers.crawler import _prepare_download_path
from cyberdrop_dl.database import Database, common, schema
from cyberdrop_dl.exceptions import DatabaseError
from cyberdrop_dl.scrape_source import _create_item_from_row
from cyberdrop_dl.url_objects import AbsoluteHttpURL, RetryInfo, ScrapeItem
from cyberdrop_dl.utils import enter_context, parse_url
_MOCK_ROW = {
"referer": "https://drive.google.com/file/d/1F0YBsnQRvrMbK0p9UlnyLu88kqQ0j_F6/edit",
"download_path": "/cdl/downloads",
"completed_at": None,
"created_at": None,
}
@pytest.fixture
def item() -> ScrapeItem:
return ScrapeItem(url=AbsoluteHttpURL("https://drive.google.com"))
@pytest.mark.parametrize(
("url", "expected"),
[
(
"https://megacloud.blog/embed-2/v3/e-1/TZb4gRkOQ642?k=1&autoPlay=1&oa=0&asi=1",
"/embed-2/v3/e-1/TZb4gRkOQ642",
),
(
"https://www.mediafire.com/file/ctppmpm7giofsgv/ADOFAI.vpk",
"ADOFAI.vpk",
),
(
"https://mega.nz/#!Ue5VRSIQ!kC2E4a4JwfWWCWYNJovGFHlbz8F",
"/#!Ue5VRSIQ!kC2E4a4JwfWWCWYNJovGFHlbz8F",
),
(
"https://mega.nz/folder/oZZxyBrY#oU4jASLPpJVvqGHJIMRcgQ/file/IYZABDGY",
"/folder/oZZxyBrY#oU4jASLPpJVvqGHJIMRcgQ/file/IYZABDGY",
),
(
"https://c.bunkr-cache.se/HwdRnHMUiWOQevCg/1df93418-5063-4e1b-851e-9470cb8fc5c6.mp4",
"/1df93418-5063-4e1b-851e-9470cb8fc5c6.mp4",
),
(
"https://e-hentai.network/h/1bb8b499a5a1a21f9e25e2c42513f310c20e83a9-115314-1280-720-wbp/keystamp=1763995200-3f6832af21;fileindex=169742365;xres=1280/1_2.webp",
"/h/1bb8b499a5a1a21f9e25e2c42513f310c20e83a9-115314-1280-720-wbp",
),
(
"https://app.koofr.net/content/links/0a00467b-2901-4213-8d71-44fad80de82d/files/get/Cyberdrop-DL.v8.4.0.zip?path=/Cyberdrop-DL.v8.4.0.zip",
"/content/links/0a00467b-2901-4213-8d71-44fad80de82d/files/get/Cyberdrop-DL.v8.4.0.zip?path=/Cyberdrop-DL.v8.4.0.zip",
),
(
"https://transfer.it/cs/g?x=yhWbjogXxRLL&n=qgxVBD5D&fn=start_linux.sh",
"/cs/g?x=yhWbjogXxRLL&n=qgxVBD5D&fn=start_linux.sh",
),
],
)
def test_create_db_path(url: str, expected: str) -> None:
crawlers = scrape_mapper.get_crawlers_mapping()
url_ = parse_url(url)
crawler = scrape_mapper._best_match(crawlers, url_.host)
assert crawler
path = crawler.__db_path__(url_)
assert path == expected
class TestGetDownloadPath:
@staticmethod
def dl_path(item: ScrapeItem) -> Path:
return _prepare_download_path(item, "cyberdrop", Config())
def test_loose_file(self, item: ScrapeItem) -> None:
assert not item.folders
assert not item.part_of_album
assert item.path == Path()
assert self.dl_path(item) == Path("downloads/Loose Files (cyberdrop)")
def test_loose_file_with_parent(self, item: ScrapeItem) -> None:
item.append_folders("a/sub/folder")
assert self.dl_path(item) == Path("downloads/a-sub-folder/Loose Files (cyberdrop)")
def test_album_file(self, item: ScrapeItem) -> None:
item.append_folders("a/sub/folder")
item.part_of_album = True
assert self.dl_path(item) == Path("downloads/a-sub-folder")
def test_retry_path(self, item: ScrapeItem) -> None:
item.append_folders("a/sub/folder")
item.part_of_album = True
item.retry_info = retry = RetryInfo(
domain="example",
url_path="",
download_path=Path("/other_path/Loose Files (example)"),
download_filename="test_file",
referer=AbsoluteHttpURL("https://example.com"),
)
assert self.dl_path(item) == Path("downloads/a-sub-folder")
with enter_context(USE_RETRY_PATH, True):
assert self.dl_path(item) == retry.download_path
def test_loose_files_no_include_domain(self, item: ScrapeItem) -> None:
item.append_folders("a/sub/folder")
config = Config()
config.subfolders.include.domain = False
path = _prepare_download_path(item, "cyberdrop", config)
assert path == Path("downloads/a-sub-folder/Loose Files")
async def test_database_creation(tmp_cwd: Path) -> None:
db_file = tmp_cwd / "test_db.db"
db = Database(db_file)
async with db:
pass
assert db.is_new
size = await aio.get_size(db_file)
assert size
assert db.schema.up_to_date
async def test_pre_allocation(tmp_cwd: Path) -> None:
db_file = tmp_cwd / "test_db.db"
async with common.connect(db_file) as db:
size = await aio.get_size(db_file)
assert size == 0
async with common.connect(db_file) as db:
await common.pre_allocate_250mb(db)
size = await aio.get_size(db_file)
assert size
assert size >= 250e6
async def test_database_version_check(tmp_cwd: Path) -> None:
db_file = tmp_cwd / "test_db.db"
db_file.touch()
async with Database(db_file).connect() as db:
await db._create_tables()
assert db.is_new
await db.conn.execute("DROP TABLE 'schema_version'")
await db.conn.commit()
async with Database(db_file).connect() as db:
assert not db.is_new
assert not db.schema.up_to_date
await db.schema.create()
assert db.schema.version is None
assert await db.schema.get_version() is None
version = schema.Version(8, 8, 8)
await db.schema.update(version)
assert not db.schema.up_to_date
assert await db.schema.get_version() == version
assert db.schema.version == version
with pytest.raises(DatabaseError):
db.schema.check_version()
async def test_db_schema_dump(tmp_cwd: Path) -> None:
db_file = tmp_cwd / "test_db.db"
async with Database(db_file) as db:
current_schema = await schema.dump(db.conn)
assert current_schema == schema.V9_15_0
def test_create_item_from_row() -> None:
referer, domain, url_path, download_path, download_filename = (
"https://example.com/a/b/c/",
"example.com",
"/a/b/c",
"/home/user/downloads",
"movie.mp4",
)
item = _create_item_from_row(locals())
assert item.url.human_repr() == referer
assert item.part_of_album is True
info = item.retry_info
assert info
assert info.domain == domain
assert info.url_path == url_path
assert info.download_path == Path(download_path)
assert info.download_filename == download_filename