Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
test: move test loading to modules (#2047)
NTFSvolumedc05c39b85eb
diff --git a/tests/crawlers/__init__.py b/tests/crawlers/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/tests/crawlers/test_cases/__init__.py b/tests/crawlers/test_cases/__init__.pynew file mode 100644index 000000000..486491173--- /dev/null+++ b/tests/crawlers/test_cases/__init__.py@@ -0,0 +1,70 @@+from __future__ import annotations++import dataclasses+import runpy+from pathlib import Path+from typing import TYPE_CHECKING, Any, NotRequired, TypedDict++from pydantic import TypeAdapter++if TYPE_CHECKING:+ from collections.abc import Sequence+++class Result(TypedDict):+ # Simplified version of media_item+ url: str+ filename: NotRequired[str | type]+ debrid_link: NotRequired[str | type | None]+ original_filename: NotRequired[str | type]+ referer: NotRequired[str | type]+ album_id: NotRequired[str | type | None]+ uploaded_at: NotRequired[int | type | None]+ download_folder: NotRequired[str | type]++[email protected](slots=True)+class CrawlerTestCase:+ domain: str+ url: str+ results: list[Result]+ description: str | None = None+ fail: bool | str | int = False+ xfail: str | None = None+ skip: str | bool = False+ count: Sequence[int] | int | None = None+ options: list[str] | None = None+ log: str | None = None++ @property+ def test_id(self) -> str:+ return f"{self.domain} - {self.url}"+++type TestData = dict[str, list[dict[str, Any]]]++_TEST_CASE_ADAPTER = TypeAdapter(CrawlerTestCase)+++def load_cases() -> TestData:+ test_data: TestData = {}+ for file in (Path(__file__).parent).iterdir():+ if not file.name.startswith("_") and file.suffix == ".py":+ module_globals = runpy.run_path(str(file), run_name=file.stem)+ if (domain := module_globals["DOMAIN"]) in test_data:+ raise RuntimeError(f"Multiple tests files for {domain}")++ test_data[domain] = module_globals["TEST_CASES"]++ return test_data+++def parse_cases(data: TestData) -> list[CrawlerTestCase]:+ test_cases: list[CrawlerTestCase] = []+ for domain, cases in sorted(data.items()):+ test_cases.extend(_TEST_CASE_ADAPTER.validate_python({"domain": domain} | case) for case in cases)+ return test_cases+++def load() -> list[CrawlerTestCase]:+ return parse_cases(load_cases())diff --git a/tests/crawlers/test_cases/4chan.py b/tests/crawlers/test_cases/fourchan.pysimilarity index 100%rename from tests/crawlers/test_cases/4chan.pyrename to tests/crawlers/test_cases/fourchan.pydiff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.pyindex 9ba085c18..37d8a8b87 100644--- a/tests/crawlers/test_crawlers.py+++ b/tests/crawlers/test_crawlers.py@@ -1,21 +1,20 @@from __future__ import annotations-import dataclassesimport re-import runpyfrom collections.abc import Callable, Generator, Sequencefrom pathlib import Path-from typing import TYPE_CHECKING, Any, NotRequired, TypedDict+from typing import TYPE_CHECKING, Anyfrom unittest import mockimport pytest-from pydantic import TypeAdapterfrom cyberdrop_dl.crawlers.crawler import compose_ep_namefrom cyberdrop_dl.scrape_mapper import ScrapeMapperfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItemfrom cyberdrop_dl.utils import parse_url+from . import test_cases+if TYPE_CHECKING:from cyberdrop_dl.crawlers.crawler import Crawlerfrom cyberdrop_dl.manager import Manager@@ -25,79 +24,26 @@ def _crawler_mock(func: str = "handle_media_item") -> mock._patch[mock.AsyncMockreturn mock.patch(f"cyberdrop_dl.crawlers.crawler.Crawler.{func}", new_callable=mock.AsyncMock)-class Result(TypedDict):- # Simplified version of media_item- url: str- filename: NotRequired[str | type]- debrid_link: NotRequired[str | type | None]- original_filename: NotRequired[str | type]- referer: NotRequired[str | type]- album_id: NotRequired[str | type | None]- uploaded_at: NotRequired[int | type | None]- download_folder: NotRequired[str | type]--[email protected](slots=True)-class CrawlerTestCase:- domain: str- url: str- results: list[Result]- description: str | None = None- fail: bool | str | int = False- xfail: str | None = None- skip: str | bool = False- count: Sequence[int] | int | None = None- options: list[str] | None = None- log: str | None = None-- @property- def test_id(self) -> str:- return f"{self.domain} - {self.url}"---type TestData = dict[str, list[dict[str, Any]]]--_TEST_CASE_ADAPTER = TypeAdapter(CrawlerTestCase)-_TEST_DATA: TestData = {}---def _load_test_cases(path: Path, test_data: TestData) -> None:- module_globals = runpy.run_path(str(path), run_name=path.stem)- if (domain := module_globals["DOMAIN"]) in test_data:- raise RuntimeError(f"Multiple tests files for {domain}")-- test_data[domain] = module_globals["TEST_CASES"]---def _load_test_data() -> TestData:- test_data: TestData = {}- for file in (Path(__file__).parent / "test_cases").iterdir():- if not file.name.startswith("_") and file.suffix == ".py":- _load_test_cases(file, test_data)-- return test_data+_TEST_DATA: test_cases.TestData = {}def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:if not _TEST_DATA:- _TEST_DATA.update(_load_test_data())+ _TEST_DATA.update(test_cases.load_cases())+if "test_case" in metafunc.fixturenames:valid_domains = set(_TEST_DATA)domains_to_tests: list[str] = getattr(metafunc.config, "test_crawlers_domains", [])for domain in domains_to_tests:assert domain in valid_domains, f"{domain = } is not a valid or has not tests defined"- all_test_cases: list[CrawlerTestCase] = []- for domain, test_cases in sorted(_TEST_DATA.items()):- if domain in domains_to_tests:- all_test_cases.extend(- _TEST_CASE_ADAPTER.validate_python({"domain": domain} | case) for case in test_cases- )+ cases = {domain: cases for domain, cases in _TEST_DATA.items() if domain in domains_to_tests}+ all_test_cases = test_cases.parse_cases(cases)metafunc.parametrize("test_case", all_test_cases, ids=lambda case: case.test_id)@pytest.mark.crawler_test_case-async def test_crawler(running_manager: Manager, test_case: CrawlerTestCase) -> None:+async def test_crawler(running_manager: Manager, test_case: test_cases.CrawlerTestCase) -> None:if test_case.skip:pytest.skip(reason=test_case.skip if isinstance(test_case.skip, str) else "")@@ -123,7 +69,7 @@ async def test_crawler(running_manager: Manager, test_case: CrawlerTestCase) ->_validate_results(crawler, test_case, results)-def _assert_n_results(test_case: CrawlerTestCase, n_results: int) -> None:+def _assert_n_results(test_case: test_cases.CrawlerTestCase, n_results: int) -> None:count = test_case.count or len(test_case.results)if isinstance(count, Sequence):assert n_results in count@@ -145,7 +91,7 @@ class _NOT_NONE: # noqa: N801, PLW1641NOT_NONE = _NOT_NONE()-def _validate_results(crawler: Crawler, test_case: CrawlerTestCase, results: list[MediaItem]) -> None:+def _validate_results(crawler: Crawler, test_case: test_cases.CrawlerTestCase, results: list[MediaItem]) -> None:expected_results = sorted(test_case.results, key=lambda x: x["url"])origin = getattr(crawler, "PRIMARY_URL", AbsoluteHttpURL("https://google.com"))for index, (expected, media_item) in enumerate(zip(expected_results, results, strict=False), 1):@@ -204,7 +150,7 @@ def _re_search(expected_value: str, result_value: str) -> re.Match[str] | None:],)async def test_direct_http_crawler(running_manager: Manager, url: str, filename: str) -> None:- test_case = CrawlerTestCase(domain="no_crawler", url=url, results=[{"url": url, "filename": filename}])+ test_case = test_cases.CrawlerTestCase(domain="no_crawler", url=url, results=[{"url": url, "filename": filename}])with _crawler_mock() as func:async with ScrapeMapper(running_manager)() as scrape_mapper:@@ -268,7 +214,7 @@ def test_public_methods_have_error_handling_wrapper() -> None:def test_load_test_data() -> None:- test_data = _load_test_data()+ test_data = test_cases.load_cases()assert len(test_data) > 20assert "dropbox" in test_dataassert type(test_data["dropbox"]) is list