Skip to content

Bulk asynchronous downloader for multiple file hosts

test: move test loading to modules (#2047)

NTFSvolumedc05c39b85eb
diff --git a/tests/crawlers/__init__.py b/tests/crawlers/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/tests/crawlers/test_cases/__init__.py b/tests/crawlers/test_cases/__init__.py
new file mode 100644
index 000000000..486491173
--- /dev/null
+++ b/tests/crawlers/test_cases/__init__.py
@@ -0,0 +1,70 @@
+from __future__ import annotations
+
+import dataclasses
+import runpy
+from pathlib import Path
+from typing import TYPE_CHECKING, Any, NotRequired, TypedDict
+
+from pydantic import TypeAdapter
+
+if TYPE_CHECKING:
+ from collections.abc import Sequence
+
+
+class Result(TypedDict):
+ # Simplified version of media_item
+ url: str
+ filename: NotRequired[str | type]
+ debrid_link: NotRequired[str | type | None]
+ original_filename: NotRequired[str | type]
+ referer: NotRequired[str | type]
+ album_id: NotRequired[str | type | None]
+ uploaded_at: NotRequired[int | type | None]
+ download_folder: NotRequired[str | type]
+
+
[email protected](slots=True)
+class CrawlerTestCase:
+ domain: str
+ url: str
+ results: list[Result]
+ description: str | None = None
+ fail: bool | str | int = False
+ xfail: str | None = None
+ skip: str | bool = False
+ count: Sequence[int] | int | None = None
+ options: list[str] | None = None
+ log: str | None = None
+
+ @property
+ def test_id(self) -> str:
+ return f"{self.domain} - {self.url}"
+
+
+type TestData = dict[str, list[dict[str, Any]]]
+
+_TEST_CASE_ADAPTER = TypeAdapter(CrawlerTestCase)
+
+
+def load_cases() -> TestData:
+ test_data: TestData = {}
+ for file in (Path(__file__).parent).iterdir():
+ if not file.name.startswith("_") and file.suffix == ".py":
+ module_globals = runpy.run_path(str(file), run_name=file.stem)
+ if (domain := module_globals["DOMAIN"]) in test_data:
+ raise RuntimeError(f"Multiple tests files for {domain}")
+
+ test_data[domain] = module_globals["TEST_CASES"]
+
+ return test_data
+
+
+def parse_cases(data: TestData) -> list[CrawlerTestCase]:
+ test_cases: list[CrawlerTestCase] = []
+ for domain, cases in sorted(data.items()):
+ test_cases.extend(_TEST_CASE_ADAPTER.validate_python({"domain": domain} | case) for case in cases)
+ return test_cases
+
+
+def load() -> list[CrawlerTestCase]:
+ return parse_cases(load_cases())
diff --git a/tests/crawlers/test_cases/4chan.py b/tests/crawlers/test_cases/fourchan.py
similarity index 100%
rename from tests/crawlers/test_cases/4chan.py
rename to tests/crawlers/test_cases/fourchan.py
diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.py
index 9ba085c18..37d8a8b87 100644
--- a/tests/crawlers/test_crawlers.py
+++ b/tests/crawlers/test_crawlers.py
@@ -1,21 +1,20 @@
from __future__ import annotations
-import dataclasses
import re
-import runpy
from collections.abc import Callable, Generator, Sequence
from pathlib import Path
-from typing import TYPE_CHECKING, Any, NotRequired, TypedDict
+from typing import TYPE_CHECKING, Any
from unittest import mock
import pytest
-from pydantic import TypeAdapter
from cyberdrop_dl.crawlers.crawler import compose_ep_name
from cyberdrop_dl.scrape_mapper import ScrapeMapper
from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem
from cyberdrop_dl.utils import parse_url
+from . import test_cases
+
if TYPE_CHECKING:
from cyberdrop_dl.crawlers.crawler import Crawler
from cyberdrop_dl.manager import Manager
@@ -25,79 +24,26 @@ def _crawler_mock(func: str = "handle_media_item") -> mock._patch[mock.AsyncMock
return mock.patch(f"cyberdrop_dl.crawlers.crawler.Crawler.{func}", new_callable=mock.AsyncMock)
-class Result(TypedDict):
- # Simplified version of media_item
- url: str
- filename: NotRequired[str | type]
- debrid_link: NotRequired[str | type | None]
- original_filename: NotRequired[str | type]
- referer: NotRequired[str | type]
- album_id: NotRequired[str | type | None]
- uploaded_at: NotRequired[int | type | None]
- download_folder: NotRequired[str | type]
-
-
[email protected](slots=True)
-class CrawlerTestCase:
- domain: str
- url: str
- results: list[Result]
- description: str | None = None
- fail: bool | str | int = False
- xfail: str | None = None
- skip: str | bool = False
- count: Sequence[int] | int | None = None
- options: list[str] | None = None
- log: str | None = None
-
- @property
- def test_id(self) -> str:
- return f"{self.domain} - {self.url}"
-
-
-type TestData = dict[str, list[dict[str, Any]]]
-
-_TEST_CASE_ADAPTER = TypeAdapter(CrawlerTestCase)
-_TEST_DATA: TestData = {}
-
-
-def _load_test_cases(path: Path, test_data: TestData) -> None:
- module_globals = runpy.run_path(str(path), run_name=path.stem)
- if (domain := module_globals["DOMAIN"]) in test_data:
- raise RuntimeError(f"Multiple tests files for {domain}")
-
- test_data[domain] = module_globals["TEST_CASES"]
-
-
-def _load_test_data() -> TestData:
- test_data: TestData = {}
- for file in (Path(__file__).parent / "test_cases").iterdir():
- if not file.name.startswith("_") and file.suffix == ".py":
- _load_test_cases(file, test_data)
-
- return test_data
+_TEST_DATA: test_cases.TestData = {}
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
if not _TEST_DATA:
- _TEST_DATA.update(_load_test_data())
+ _TEST_DATA.update(test_cases.load_cases())
+
if "test_case" in metafunc.fixturenames:
valid_domains = set(_TEST_DATA)
domains_to_tests: list[str] = getattr(metafunc.config, "test_crawlers_domains", [])
for domain in domains_to_tests:
assert domain in valid_domains, f"{domain = } is not a valid or has not tests defined"
- all_test_cases: list[CrawlerTestCase] = []
- for domain, test_cases in sorted(_TEST_DATA.items()):
- if domain in domains_to_tests:
- all_test_cases.extend(
- _TEST_CASE_ADAPTER.validate_python({"domain": domain} | case) for case in test_cases
- )
+ cases = {domain: cases for domain, cases in _TEST_DATA.items() if domain in domains_to_tests}
+ all_test_cases = test_cases.parse_cases(cases)
metafunc.parametrize("test_case", all_test_cases, ids=lambda case: case.test_id)
@pytest.mark.crawler_test_case
-async def test_crawler(running_manager: Manager, test_case: CrawlerTestCase) -> None:
+async def test_crawler(running_manager: Manager, test_case: test_cases.CrawlerTestCase) -> None:
if test_case.skip:
pytest.skip(reason=test_case.skip if isinstance(test_case.skip, str) else "")
@@ -123,7 +69,7 @@ async def test_crawler(running_manager: Manager, test_case: CrawlerTestCase) ->
_validate_results(crawler, test_case, results)
-def _assert_n_results(test_case: CrawlerTestCase, n_results: int) -> None:
+def _assert_n_results(test_case: test_cases.CrawlerTestCase, n_results: int) -> None:
count = test_case.count or len(test_case.results)
if isinstance(count, Sequence):
assert n_results in count
@@ -145,7 +91,7 @@ class _NOT_NONE: # noqa: N801, PLW1641
NOT_NONE = _NOT_NONE()
-def _validate_results(crawler: Crawler, test_case: CrawlerTestCase, results: list[MediaItem]) -> None:
+def _validate_results(crawler: Crawler, test_case: test_cases.CrawlerTestCase, results: list[MediaItem]) -> None:
expected_results = sorted(test_case.results, key=lambda x: x["url"])
origin = getattr(crawler, "PRIMARY_URL", AbsoluteHttpURL("https://google.com"))
for index, (expected, media_item) in enumerate(zip(expected_results, results, strict=False), 1):
@@ -204,7 +150,7 @@ def _re_search(expected_value: str, result_value: str) -> re.Match[str] | None:
],
)
async def test_direct_http_crawler(running_manager: Manager, url: str, filename: str) -> None:
- test_case = CrawlerTestCase(domain="no_crawler", url=url, results=[{"url": url, "filename": filename}])
+ test_case = test_cases.CrawlerTestCase(domain="no_crawler", url=url, results=[{"url": url, "filename": filename}])
with _crawler_mock() as func:
async with ScrapeMapper(running_manager)() as scrape_mapper:
@@ -268,7 +214,7 @@ def test_public_methods_have_error_handling_wrapper() -> None:
def test_load_test_data() -> None:
- test_data = _load_test_data()
+ test_data = test_cases.load_cases()
assert len(test_data) > 20
assert "dropbox" in test_data
assert type(test_data["dropbox"]) is list