Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
tests: add crawler tests
NTFSvolume604d2bd954ce
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yamlindex e2c4c98d5..ad6a75cdd 100644--- a/.pre-commit-config.yaml+++ b/.pre-commit-config.yaml@@ -11,8 +11,6 @@ repos:- id: check-yaml- id: detect-private-key- id: end-of-file-fixer- - id: name-tests-test- args: [ --pytest-test-first ]- id: trailing-whitespaceargs: [ --markdown-linebreak-ext=md ]diff --git a/pyproject.toml b/pyproject.tomlindex 41779dc81..0aa4ce58c 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -82,6 +82,9 @@ pytest-mock = "*"addopts = ["-s"]asyncio_default_fixture_loop_scope = "function"asyncio_mode = "auto"+markers = [+ "crawler_test_case: tests that do full run with a crawler (making actual network requests)"+]minversion = "8.4.1"testpaths = ["tests"]diff --git a/tests/conftest.py b/tests/conftest.pyindex 42203bceb..e30ef3ad7 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -5,9 +5,51 @@ from typing import TYPE_CHECKINGimport pytest+from cyberdrop_dl.managers.manager import Manager+from cyberdrop_dl.scraper import scrape_mapper+if TYPE_CHECKING:+ from collections.abc import AsyncGeneratorfrom pathlib import Path+ class Config(pytest.Config): # type: ignore+ test_crawlers_domains: set[str]+++def pytest_addoption(parser: pytest.Parser):+ parser.addoption(+ "--test-crawlers",+ action="store",+ help="A comma-separated list of crawlers' domains (e.g., 'dropbox.com,jpg5.su').",+ default="",+ )+++def pytest_configure(config: Config):+ config.test_crawlers_domains = {+ domain for item in config.getoption("--test-crawlers").split(",") if (domain := item.strip())+ }+++def pytest_collection_modifyitems(config: Config, items: list[pytest.Item]) -> None:+ """When running with --test-crawlers, disable all other tests"""+ if not config.test_crawlers_domains:+ return++ selected_tests = []+ deselected_tests = []+ for item in items:+ markers = {marker.name for marker in item.iter_markers()}++ if "crawler_test_case" in markers:+ selected_tests.append(item)+ else:+ deselected_tests.append(item)++ if deselected_tests:+ config.hook.pytest_deselected(items=deselected_tests)+ items[:] = selected_tests+@pytest.fixturedef tmp_cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:@@ -19,3 +61,25 @@ def tmp_cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:async def logs(caplog: pytest.LogCaptureFixture) -> pytest.LogCaptureFixture:caplog.set_level(10)return caplog++[email protected](scope="function", name="manager")+def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Manager:+ appdata = str(tmp_path)+ downloads = str(tmp_path / "Downloads")+ monkeypatch.chdir(tmp_path)+ bare_manager = Manager(("--appdata-folder", appdata, "-d", downloads))+ bare_manager.startup()+ bare_manager.path_manager.startup()+ bare_manager.log_manager.startup()+ return bare_manager++[email protected](scope="function")+async def running_manager(manager: Manager) -> AsyncGenerator[Manager]:+ scrape_mapper.existing_crawlers.clear()+ await manager.async_startup()+ manager.states.RUNNING.set()+ yield manager+ manager.states.RUNNING.clear()+ await manager.close()diff --git a/tests/crawlers/test_cases/jpg5.py b/tests/crawlers/test_cases/jpg5.pynew file mode 100644index 000000000..e9bd03540--- /dev/null+++ b/tests/crawlers/test_cases/jpg5.py@@ -0,0 +1,45 @@+DOMAIN = "jpg5.su"+TEST_CASES = [+ (+ "https://jpg6.su/img/00612-3170878818-raw-photo-nude-woman-naked-braless-no-cloth.Yvjr1u7",+ [+ {+ "url": "https://simp4.jpg5.su/00612-3170878818-RAW-photo-of-a-nude-woman-naked_-braless-no-clothes-highly-detailed-skin-8k-uhd-medium-breast-size-detailed-nipples6b05c0ef6025d0a0.jpg",+ "filename": "00612-3170878818-RAW-photo-of-a-nude-woman-naked_-braless-no-clothes-highly-detailed-skin-8k-uh.jpg",+ "referer": "https://jpg6.su/img/Yvjr1u7",+ "album_id": None,+ "datetime": 1693206885,+ }+ ],+ ),+ (+ "https://jpg6.su/img/2520x3360-ab219efe5d20f99740d58188edd20c440659035b8a5c979d.N3iJfxa",+ [+ {+ "url": "https://simp6.selti-delivery.ru/images3/2520x3360_ab219efe5d20f99740d58188edd20c440659035b8a5c979d2f40a0e086780c7d.jpg",+ "filename": "2520x3360_ab219efe5d20f99740d58188edd20c440659035b8a5c979d2f40a0e086780c7d.jpg",+ "referer": "https://jpg6.su/img/N3iJfxa",+ "datetime": 1753502011,+ }+ ],+ ),+ (+ "https://jpg6.su/a/testalbum.YnfD4p",+ [+ {+ "url": "https://simp6.selti-delivery.ru/images3/landscape-mountains-nature-5564166880341c46b7abbe.jpg",+ "filename": "landscape-mountains-nature-5564166880341c46b7abbe.jpg",+ "referer": "https://jpg6.su/img/N3TGEr1",+ "datetime": None,+ "album_id": "YnfD4p",+ },+ {+ "url": "https://simp6.selti-delivery.ru/images3/47749882ae05d0349d87f2d.jpg",+ "filename": "47749882ae05d0349d87f2d.jpg",+ "referer": "https://jpg6.su/img/N3TGCdA",+ "datetime": None,+ "album_id": "YnfD4p",+ },+ ],+ ),+]diff --git a/tests/crawlers/test_cases/pixhost.py b/tests/crawlers/test_cases/pixhost.pynew file mode 100644index 000000000..51b49f185--- /dev/null+++ b/tests/crawlers/test_cases/pixhost.py@@ -0,0 +1,14 @@+DOMAIN = "pixhost"+TEST_CASES = [+ (+ "https://pixhost.to/show/491/538303562_035.jpg",+ [+ {+ "url": "https://img100.pixhost.to/images/491/538303562_035.jpg",+ "filename": "538303562_035.jpg",+ "referer": "https://pixhost.to/show/491/538303562_035.jpg",+ "album_id": None, # TODO: get the album id for individual images+ },+ ],+ ),+]diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.pynew file mode 100644index 000000000..eea64200e--- /dev/null+++ b/tests/crawlers/test_crawlers.py@@ -0,0 +1,108 @@+from __future__ import annotations++import importlib.util+from pathlib import Path+from typing import TYPE_CHECKING, Literal, NamedTuple, NotRequired+from unittest import mock++import pytest+from pydantic import TypeAdapter+from typing_extensions import TypedDict++from cyberdrop_dl.data_structures.url_objects import MediaItem, ScrapeItem+from cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper++if TYPE_CHECKING:+ from cyberdrop_dl.crawlers.crawler import Crawler+ from cyberdrop_dl.managers.manager import Manager+++def _amock(crawler: Crawler, func: str = "handle_media_item") -> mock._patch[mock.AsyncMock]:+ return mock.patch.object(crawler, func, new_callable=mock.AsyncMock)+++class Result(TypedDict):+ # Simplified version of media_item+ url: str+ filename: str+ debrid_link: NotRequired[Literal["ANY"] | None]+ original_filename: NotRequired[str]+ referer: NotRequired[str]+ album_id: NotRequired[str | None]+ datetime: NotRequired[int | None]+++class CrawlerTestCase(NamedTuple):+ domain: str+ input_url: str+ results: list[Result]+++_TEST_CASE_ADAPTER = TypeAdapter(CrawlerTestCase)+_TEST_DATA: dict[str, list[tuple[str, list[Result]]]] = {}+++def _load_test_cases(path: Path) -> None:+ module_spec = importlib.util.spec_from_file_location(path.stem, path)+ assert module_spec and module_spec.loader+ module = importlib.util.module_from_spec(module_spec)+ module_spec.loader.exec_module(module)+ _TEST_DATA[module.DOMAIN] = module.TEST_CASES+++def _load_test_data() -> None:+ if _TEST_DATA:+ return+ for file in (Path(__file__).parent / "test_cases").iterdir():+ if not file.name.startswith("_") and file.suffix == ".py":+ _load_test_cases(file)+++def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:+ _load_test_data()+ if "crawler_test_case" in metafunc.fixturenames:+ valid_domains = sorted(_TEST_DATA)+ domains_to_tests: list[str] = metafunc.config.test_crawlers_domains # type: ignore+ for domain in domains_to_tests:+ assert domain in valid_domains, f"{domain = } is not a valid or has not tests defined"++ all_test_cases: list[CrawlerTestCase] = []+ for domain, test_cases in _TEST_DATA.items():+ if domain in domains_to_tests:+ all_test_cases.extend(CrawlerTestCase(domain, *case) for case in test_cases)+ metafunc.parametrize("crawler_test_case", all_test_cases, ids=lambda x: f"{x.domain} - {x.input_url}")++[email protected]_test_case+async def test_crawler(running_manager: Manager, crawler_test_case: CrawlerTestCase) -> None:+ # Check that this is a valid test case with pydantic+ domain, input_url, expected_results = _TEST_CASE_ADAPTER.validate_python(crawler_test_case, strict=True)++ async with ScrapeMapper(running_manager) as scrape_mapper:+ await scrape_mapper.run()+ crawler = next(+ (crawler for crawler in scrape_mapper.existing_crawlers.values() if crawler.DOMAIN == domain), None+ )+ assert crawler, f"{domain} is not a valid crawler domain. Test case is invalid"+ await crawler.startup()+ item = ScrapeItem(url=crawler.parse_url(input_url))+ with _amock(crawler) as func:+ await crawler.run(item)+ results: list[MediaItem] = sorted((call.args[0] for call in func.call_args_list), key=lambda x: x.url)++ expected_results = sorted(expected_results, key=lambda x: x["url"])+ _validate_results(crawler, expected_results, results)+++def _validate_results(crawler: Crawler, expected_results: list[Result], results: list[MediaItem]) -> None:+ assert len(expected_results) == len(results)+ for expected_result, media_item in zip(expected_results, results, strict=True):+ for attr_name, expected_value in expected_result.items():+ result_value = getattr(media_item, attr_name)+ if isinstance(expected_value, str):+ if expected_value.startswith("http"):+ expected_value = crawler.parse_url(expected_value)+ elif expected_value == "ANY":+ expected_value = mock.ANY++ assert expected_value == result_value, f"{attr_name} is different"diff --git a/tests/test_manager.py b/tests/test_manager.pyindex 7ea111c0c..42fdd84aa 100644--- a/tests/test_manager.py+++ b/tests/test_manager.py@@ -1,31 +1,22 @@+from __future__ import annotations+from dataclasses import Field-from pathlib import Path-from typing import Any, TypeVar+from typing import TYPE_CHECKING, Any, TypeVarimport pytest-from pydantic import BaseModelfrom cyberdrop_dl.managers.manager import Manager, merge_dicts-M = TypeVar("M", bound=BaseModel)+if TYPE_CHECKING:+ from pydantic import BaseModel++ M = TypeVar("M", bound=BaseModel)def update_model(model: M, **kwargs: Any) -> M:return model.model_validate(model.model_dump() | kwargs)[email protected](scope="function", name="manager")-def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Manager:- appdata = str(tmp_path)- downloads = str(tmp_path / "Downloads")- monkeypatch.chdir(tmp_path)- bare_manager = Manager(("--appdata-folder", appdata, "-d", downloads))- bare_manager.startup()- bare_manager.path_manager.startup()- bare_manager.log_manager.startup()- return bare_manager--class TestMergeDicts:def test_overwrite(self) -> None:dict1 = {"a": 1, "b": 2}@@ -112,11 +103,11 @@ class TestMergeDicts:],)def test_args_logging_should_censor_webhook(- manager: Manager, logs: pytest.LogCaptureFixture, webhook: str, output: str+ running_manager: Manager, logs: pytest.LogCaptureFixture, webhook: str, output: str) -> None:- logs_model = manager.config_manager.settings_data.logs- manager.config_manager.settings_data.logs = update_model(logs_model, webhook=webhook)- manager.args_logging()+ logs_model = running_manager.config_manager.settings_data.logs+ running_manager.config_manager.settings_data.logs = update_model(logs_model, webhook=webhook)+ running_manager.args_logging()assert logs.messagesassert "Starting Cyberdrop-DL Process" in logs.textassert webhook not in logs.text@@ -126,12 +117,12 @@ def test_args_logging_should_censor_webhook(assert output == webhook_url-async def test_async_db_close(manager: Manager) -> None:- await manager.async_startup()- assert not isinstance(manager.db_manager, Field)- assert not isinstance(manager.hash_manager, Field)- assert "overwrite" not in str(manager.log_manager.main_log)- await manager.async_db_close()- assert isinstance(manager.db_manager, Field)- assert isinstance(manager.hash_manager, Field)- await manager.close()+async def test_async_db_close(running_manager: Manager) -> None:+ await running_manager.async_startup()+ assert not isinstance(running_manager.db_manager, Field)+ assert not isinstance(running_manager.hash_manager, Field)+ assert "overwrite" not in str(running_manager.log_manager.main_log)+ await running_manager.async_db_close()+ assert isinstance(running_manager.db_manager, Field)+ assert isinstance(running_manager.hash_manager, Field)+ await running_manager.close()diff --git a/tests/test_scrape_mapper.py b/tests/test_scrape_mapper.pyindex 28d11757e..d1ebdff54 100644--- a/tests/test_scrape_mapper.py+++ b/tests/test_scrape_mapper.py@@ -1,23 +1,19 @@+from __future__ import annotations++from typing import TYPE_CHECKING+import pytestfrom cyberdrop_dl import crawlersfrom cyberdrop_dl.crawlers.crawler import create_crawlers-from cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.scraper import scrape_mapper-_ = scrape_mapper.get_crawlers_mapping()+if TYPE_CHECKING:+ from cyberdrop_dl.managers.manager import ManagerTEST_BASE_CRAWLER = next(iter(crawlers.GENERIC_CRAWLERS))-class MockProgress:- scraping_progress = None---manager = Manager()-manager.progress_manager = MockProgress() # type: ignore--@pytest.mark.parametrize("link",[@@ -29,10 +25,12 @@ manager.progress_manager = MockProgress() # type: ignore"https://forums.socialmediagirls.com/threads/en-fr-tools-to-download-upload-content-websites-softwares-extensions.13930/#post-2070848",],)-def test_generic_crawlers_that_match_supported_crawlers_should_not_be_created(link: str) -> None:+def test_generic_crawlers_that_match_supported_crawlers_should_not_be_created(+ running_manager: Manager, link: str+) -> None:crawler = next(iter(create_crawlers([link], TEST_BASE_CRAWLER)))with pytest.raises(ValueError) as exc_info:- scrape_mapper.register_crawler(scrape_mapper.existing_crawlers, crawler(manager), from_user="raise")+ scrape_mapper.register_crawler(scrape_mapper.existing_crawlers, crawler(running_manager), from_user="raise")assert f"Unable to assign {link.split('/')[0]}" in str(exc_info.value)@@ -44,11 +42,13 @@ def test_generic_crawlers_that_match_supported_crawlers_should_not_be_created(li"https://meta.discourse.org/",],)-def test_generic_crawlers_that_do_no_match_supported_crawlers_should_be_created(link: str) -> None:+def test_generic_crawlers_that_do_no_match_supported_crawlers_should_be_created(+ running_manager: Manager, link: str+) -> None:crawler = next(iter(create_crawlers([link], TEST_BASE_CRAWLER)))existing_crawlers = scrape_mapper.existing_crawlers.copy()crawlers_before = set(existing_crawlers.values())- scrape_mapper.register_crawler(existing_crawlers, crawler(manager), from_user="raise")+ scrape_mapper.register_crawler(existing_crawlers, crawler(running_manager), from_user="raise")new_crawlers = set(existing_crawlers.values()) - crawlers_beforeassert len(new_crawlers) == 1created_crawler = next(iter(new_crawlers))