Skip to content

Bulk asynchronous downloader for multiple file hosts

tests: add crawler tests

NTFSvolume604d2bd954ce
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index e2c4c98d5..ad6a75cdd 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -11,8 +11,6 @@ repos:
- id: check-yaml
- id: detect-private-key
- id: end-of-file-fixer
- - id: name-tests-test
- args: [ --pytest-test-first ]
- id: trailing-whitespace
args: [ --markdown-linebreak-ext=md ]
diff --git a/pyproject.toml b/pyproject.toml
index 41779dc81..0aa4ce58c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -82,6 +82,9 @@ pytest-mock = "*"
addopts = ["-s"]
asyncio_default_fixture_loop_scope = "function"
asyncio_mode = "auto"
+markers = [
+ "crawler_test_case: tests that do full run with a crawler (making actual network requests)"
+]
minversion = "8.4.1"
testpaths = ["tests"]
diff --git a/tests/conftest.py b/tests/conftest.py
index 42203bceb..e30ef3ad7 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -5,9 +5,51 @@ from typing import TYPE_CHECKING
import pytest
+from cyberdrop_dl.managers.manager import Manager
+from cyberdrop_dl.scraper import scrape_mapper
+
if TYPE_CHECKING:
+ from collections.abc import AsyncGenerator
from pathlib import Path
+ class Config(pytest.Config): # type: ignore
+ test_crawlers_domains: set[str]
+
+
+def pytest_addoption(parser: pytest.Parser):
+ parser.addoption(
+ "--test-crawlers",
+ action="store",
+ help="A comma-separated list of crawlers' domains (e.g., 'dropbox.com,jpg5.su').",
+ default="",
+ )
+
+
+def pytest_configure(config: Config):
+ config.test_crawlers_domains = {
+ domain for item in config.getoption("--test-crawlers").split(",") if (domain := item.strip())
+ }
+
+
+def pytest_collection_modifyitems(config: Config, items: list[pytest.Item]) -> None:
+ """When running with --test-crawlers, disable all other tests"""
+ if not config.test_crawlers_domains:
+ return
+
+ selected_tests = []
+ deselected_tests = []
+ for item in items:
+ markers = {marker.name for marker in item.iter_markers()}
+
+ if "crawler_test_case" in markers:
+ selected_tests.append(item)
+ else:
+ deselected_tests.append(item)
+
+ if deselected_tests:
+ config.hook.pytest_deselected(items=deselected_tests)
+ items[:] = selected_tests
+
@pytest.fixture
def tmp_cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
@@ -19,3 +61,25 @@ def tmp_cwd(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
async def logs(caplog: pytest.LogCaptureFixture) -> pytest.LogCaptureFixture:
caplog.set_level(10)
return caplog
+
+
[email protected](scope="function", name="manager")
+def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Manager:
+ appdata = str(tmp_path)
+ downloads = str(tmp_path / "Downloads")
+ monkeypatch.chdir(tmp_path)
+ bare_manager = Manager(("--appdata-folder", appdata, "-d", downloads))
+ bare_manager.startup()
+ bare_manager.path_manager.startup()
+ bare_manager.log_manager.startup()
+ return bare_manager
+
+
[email protected](scope="function")
+async def running_manager(manager: Manager) -> AsyncGenerator[Manager]:
+ scrape_mapper.existing_crawlers.clear()
+ await manager.async_startup()
+ manager.states.RUNNING.set()
+ yield manager
+ manager.states.RUNNING.clear()
+ await manager.close()
diff --git a/tests/crawlers/test_cases/jpg5.py b/tests/crawlers/test_cases/jpg5.py
new file mode 100644
index 000000000..e9bd03540
--- /dev/null
+++ b/tests/crawlers/test_cases/jpg5.py
@@ -0,0 +1,45 @@
+DOMAIN = "jpg5.su"
+TEST_CASES = [
+ (
+ "https://jpg6.su/img/00612-3170878818-raw-photo-nude-woman-naked-braless-no-cloth.Yvjr1u7",
+ [
+ {
+ "url": "https://simp4.jpg5.su/00612-3170878818-RAW-photo-of-a-nude-woman-naked_-braless-no-clothes-highly-detailed-skin-8k-uhd-medium-breast-size-detailed-nipples6b05c0ef6025d0a0.jpg",
+ "filename": "00612-3170878818-RAW-photo-of-a-nude-woman-naked_-braless-no-clothes-highly-detailed-skin-8k-uh.jpg",
+ "referer": "https://jpg6.su/img/Yvjr1u7",
+ "album_id": None,
+ "datetime": 1693206885,
+ }
+ ],
+ ),
+ (
+ "https://jpg6.su/img/2520x3360-ab219efe5d20f99740d58188edd20c440659035b8a5c979d.N3iJfxa",
+ [
+ {
+ "url": "https://simp6.selti-delivery.ru/images3/2520x3360_ab219efe5d20f99740d58188edd20c440659035b8a5c979d2f40a0e086780c7d.jpg",
+ "filename": "2520x3360_ab219efe5d20f99740d58188edd20c440659035b8a5c979d2f40a0e086780c7d.jpg",
+ "referer": "https://jpg6.su/img/N3iJfxa",
+ "datetime": 1753502011,
+ }
+ ],
+ ),
+ (
+ "https://jpg6.su/a/testalbum.YnfD4p",
+ [
+ {
+ "url": "https://simp6.selti-delivery.ru/images3/landscape-mountains-nature-5564166880341c46b7abbe.jpg",
+ "filename": "landscape-mountains-nature-5564166880341c46b7abbe.jpg",
+ "referer": "https://jpg6.su/img/N3TGEr1",
+ "datetime": None,
+ "album_id": "YnfD4p",
+ },
+ {
+ "url": "https://simp6.selti-delivery.ru/images3/47749882ae05d0349d87f2d.jpg",
+ "filename": "47749882ae05d0349d87f2d.jpg",
+ "referer": "https://jpg6.su/img/N3TGCdA",
+ "datetime": None,
+ "album_id": "YnfD4p",
+ },
+ ],
+ ),
+]
diff --git a/tests/crawlers/test_cases/pixhost.py b/tests/crawlers/test_cases/pixhost.py
new file mode 100644
index 000000000..51b49f185
--- /dev/null
+++ b/tests/crawlers/test_cases/pixhost.py
@@ -0,0 +1,14 @@
+DOMAIN = "pixhost"
+TEST_CASES = [
+ (
+ "https://pixhost.to/show/491/538303562_035.jpg",
+ [
+ {
+ "url": "https://img100.pixhost.to/images/491/538303562_035.jpg",
+ "filename": "538303562_035.jpg",
+ "referer": "https://pixhost.to/show/491/538303562_035.jpg",
+ "album_id": None, # TODO: get the album id for individual images
+ },
+ ],
+ ),
+]
diff --git a/tests/crawlers/test_crawlers.py b/tests/crawlers/test_crawlers.py
new file mode 100644
index 000000000..eea64200e
--- /dev/null
+++ b/tests/crawlers/test_crawlers.py
@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+from typing import TYPE_CHECKING, Literal, NamedTuple, NotRequired
+from unittest import mock
+
+import pytest
+from pydantic import TypeAdapter
+from typing_extensions import TypedDict
+
+from cyberdrop_dl.data_structures.url_objects import MediaItem, ScrapeItem
+from cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.crawlers.crawler import Crawler
+ from cyberdrop_dl.managers.manager import Manager
+
+
+def _amock(crawler: Crawler, func: str = "handle_media_item") -> mock._patch[mock.AsyncMock]:
+ return mock.patch.object(crawler, func, new_callable=mock.AsyncMock)
+
+
+class Result(TypedDict):
+ # Simplified version of media_item
+ url: str
+ filename: str
+ debrid_link: NotRequired[Literal["ANY"] | None]
+ original_filename: NotRequired[str]
+ referer: NotRequired[str]
+ album_id: NotRequired[str | None]
+ datetime: NotRequired[int | None]
+
+
+class CrawlerTestCase(NamedTuple):
+ domain: str
+ input_url: str
+ results: list[Result]
+
+
+_TEST_CASE_ADAPTER = TypeAdapter(CrawlerTestCase)
+_TEST_DATA: dict[str, list[tuple[str, list[Result]]]] = {}
+
+
+def _load_test_cases(path: Path) -> None:
+ module_spec = importlib.util.spec_from_file_location(path.stem, path)
+ assert module_spec and module_spec.loader
+ module = importlib.util.module_from_spec(module_spec)
+ module_spec.loader.exec_module(module)
+ _TEST_DATA[module.DOMAIN] = module.TEST_CASES
+
+
+def _load_test_data() -> None:
+ if _TEST_DATA:
+ return
+ for file in (Path(__file__).parent / "test_cases").iterdir():
+ if not file.name.startswith("_") and file.suffix == ".py":
+ _load_test_cases(file)
+
+
+def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
+ _load_test_data()
+ if "crawler_test_case" in metafunc.fixturenames:
+ valid_domains = sorted(_TEST_DATA)
+ domains_to_tests: list[str] = metafunc.config.test_crawlers_domains # type: ignore
+ for domain in domains_to_tests:
+ assert domain in valid_domains, f"{domain = } is not a valid or has not tests defined"
+
+ all_test_cases: list[CrawlerTestCase] = []
+ for domain, test_cases in _TEST_DATA.items():
+ if domain in domains_to_tests:
+ all_test_cases.extend(CrawlerTestCase(domain, *case) for case in test_cases)
+ metafunc.parametrize("crawler_test_case", all_test_cases, ids=lambda x: f"{x.domain} - {x.input_url}")
+
+
+async def test_crawler(running_manager: Manager, crawler_test_case: CrawlerTestCase) -> None:
+ # Check that this is a valid test case with pydantic
+ domain, input_url, expected_results = _TEST_CASE_ADAPTER.validate_python(crawler_test_case, strict=True)
+
+ async with ScrapeMapper(running_manager) as scrape_mapper:
+ await scrape_mapper.run()
+ crawler = next(
+ (crawler for crawler in scrape_mapper.existing_crawlers.values() if crawler.DOMAIN == domain), None
+ )
+ assert crawler, f"{domain} is not a valid crawler domain. Test case is invalid"
+ await crawler.startup()
+ item = ScrapeItem(url=crawler.parse_url(input_url))
+ with _amock(crawler) as func:
+ await crawler.run(item)
+ results: list[MediaItem] = sorted((call.args[0] for call in func.call_args_list), key=lambda x: x.url)
+
+ expected_results = sorted(expected_results, key=lambda x: x["url"])
+ _validate_results(crawler, expected_results, results)
+
+
+def _validate_results(crawler: Crawler, expected_results: list[Result], results: list[MediaItem]) -> None:
+ assert len(expected_results) == len(results)
+ for expected_result, media_item in zip(expected_results, results, strict=True):
+ for attr_name, expected_value in expected_result.items():
+ result_value = getattr(media_item, attr_name)
+ if isinstance(expected_value, str):
+ if expected_value.startswith("http"):
+ expected_value = crawler.parse_url(expected_value)
+ elif expected_value == "ANY":
+ expected_value = mock.ANY
+
+ assert expected_value == result_value, f"{attr_name} is different"
diff --git a/tests/test_manager.py b/tests/test_manager.py
index 7ea111c0c..42fdd84aa 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -1,31 +1,22 @@
+from __future__ import annotations
+
from dataclasses import Field
-from pathlib import Path
-from typing import Any, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar
import pytest
-from pydantic import BaseModel
from cyberdrop_dl.managers.manager import Manager, merge_dicts
-M = TypeVar("M", bound=BaseModel)
+if TYPE_CHECKING:
+ from pydantic import BaseModel
+
+ M = TypeVar("M", bound=BaseModel)
def update_model(model: M, **kwargs: Any) -> M:
return model.model_validate(model.model_dump() | kwargs)
[email protected](scope="function", name="manager")
-def post_startup_manager(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Manager:
- appdata = str(tmp_path)
- downloads = str(tmp_path / "Downloads")
- monkeypatch.chdir(tmp_path)
- bare_manager = Manager(("--appdata-folder", appdata, "-d", downloads))
- bare_manager.startup()
- bare_manager.path_manager.startup()
- bare_manager.log_manager.startup()
- return bare_manager
-
-
class TestMergeDicts:
def test_overwrite(self) -> None:
dict1 = {"a": 1, "b": 2}
@@ -112,11 +103,11 @@ class TestMergeDicts:
],
)
def test_args_logging_should_censor_webhook(
- manager: Manager, logs: pytest.LogCaptureFixture, webhook: str, output: str
+ running_manager: Manager, logs: pytest.LogCaptureFixture, webhook: str, output: str
) -> None:
- logs_model = manager.config_manager.settings_data.logs
- manager.config_manager.settings_data.logs = update_model(logs_model, webhook=webhook)
- manager.args_logging()
+ logs_model = running_manager.config_manager.settings_data.logs
+ running_manager.config_manager.settings_data.logs = update_model(logs_model, webhook=webhook)
+ running_manager.args_logging()
assert logs.messages
assert "Starting Cyberdrop-DL Process" in logs.text
assert webhook not in logs.text
@@ -126,12 +117,12 @@ def test_args_logging_should_censor_webhook(
assert output == webhook_url
-async def test_async_db_close(manager: Manager) -> None:
- await manager.async_startup()
- assert not isinstance(manager.db_manager, Field)
- assert not isinstance(manager.hash_manager, Field)
- assert "overwrite" not in str(manager.log_manager.main_log)
- await manager.async_db_close()
- assert isinstance(manager.db_manager, Field)
- assert isinstance(manager.hash_manager, Field)
- await manager.close()
+async def test_async_db_close(running_manager: Manager) -> None:
+ await running_manager.async_startup()
+ assert not isinstance(running_manager.db_manager, Field)
+ assert not isinstance(running_manager.hash_manager, Field)
+ assert "overwrite" not in str(running_manager.log_manager.main_log)
+ await running_manager.async_db_close()
+ assert isinstance(running_manager.db_manager, Field)
+ assert isinstance(running_manager.hash_manager, Field)
+ await running_manager.close()
diff --git a/tests/test_scrape_mapper.py b/tests/test_scrape_mapper.py
index 28d11757e..d1ebdff54 100644
--- a/tests/test_scrape_mapper.py
+++ b/tests/test_scrape_mapper.py
@@ -1,23 +1,19 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
import pytest
from cyberdrop_dl import crawlers
from cyberdrop_dl.crawlers.crawler import create_crawlers
-from cyberdrop_dl.managers.manager import Manager
from cyberdrop_dl.scraper import scrape_mapper
-_ = scrape_mapper.get_crawlers_mapping()
+if TYPE_CHECKING:
+ from cyberdrop_dl.managers.manager import Manager
TEST_BASE_CRAWLER = next(iter(crawlers.GENERIC_CRAWLERS))
-class MockProgress:
- scraping_progress = None
-
-
-manager = Manager()
-manager.progress_manager = MockProgress() # type: ignore
-
-
@pytest.mark.parametrize(
"link",
[
@@ -29,10 +25,12 @@ manager.progress_manager = MockProgress() # type: ignore
"https://forums.socialmediagirls.com/threads/en-fr-tools-to-download-upload-content-websites-softwares-extensions.13930/#post-2070848",
],
)
-def test_generic_crawlers_that_match_supported_crawlers_should_not_be_created(link: str) -> None:
+def test_generic_crawlers_that_match_supported_crawlers_should_not_be_created(
+ running_manager: Manager, link: str
+) -> None:
crawler = next(iter(create_crawlers([link], TEST_BASE_CRAWLER)))
with pytest.raises(ValueError) as exc_info:
- scrape_mapper.register_crawler(scrape_mapper.existing_crawlers, crawler(manager), from_user="raise")
+ scrape_mapper.register_crawler(scrape_mapper.existing_crawlers, crawler(running_manager), from_user="raise")
assert f"Unable to assign {link.split('/')[0]}" in str(exc_info.value)
@@ -44,11 +42,13 @@ def test_generic_crawlers_that_match_supported_crawlers_should_not_be_created(li
"https://meta.discourse.org/",
],
)
-def test_generic_crawlers_that_do_no_match_supported_crawlers_should_be_created(link: str) -> None:
+def test_generic_crawlers_that_do_no_match_supported_crawlers_should_be_created(
+ running_manager: Manager, link: str
+) -> None:
crawler = next(iter(create_crawlers([link], TEST_BASE_CRAWLER)))
existing_crawlers = scrape_mapper.existing_crawlers.copy()
crawlers_before = set(existing_crawlers.values())
- scrape_mapper.register_crawler(existing_crawlers, crawler(manager), from_user="raise")
+ scrape_mapper.register_crawler(existing_crawlers, crawler(running_manager), from_user="raise")
new_crawlers = set(existing_crawlers.values()) - crawlers_before
assert len(new_crawlers) == 1
created_crawler = next(iter(new_crawlers))