Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: use crawler registry for everything (#86)

NTFSvolumee86eb2f09042
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index a34babaea..e69de29bb 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -1,169 +0,0 @@
-# ruff: noqa: F401
-from __future__ import annotations
-
-from cyberdrop_dl import env
-
-from ._chevereto import CheveretoCrawler
-from .anontransfer import AnonTransferCrawler
-from .anysex import AnySexCrawler
-from .archivebate import ArchiveBateCrawler
-from .ashemaletube import AShemaleTubeCrawler
-from .bandcamp import BandcampCrawler
-from .beeg import BeegComCrawler
-from .box_dot_com import BoxDotComCrawler
-from .bunkrr import BunkrrCrawler
-from .bunkrr_albums_io import BunkrAlbumsIOCrawler
-from .buzzheavier import BuzzHeavierCrawler
-from .camwhores_dot_tv import CamwhoresTVCrawler
-from .catbox import CatboxCrawler
-from .cloud_mail_ru import CloudMailRuCrawler
-from .cloudflare_stream import CloudflareStreamCrawler
-from .coomer import CoomerCrawler
-from .crawler import Crawler
-from .cyberdrop import CyberdropCrawler
-from .cyberfile import CyberfileCrawler
-from .desivideo import DesiVideoCrawler
-from .dirtyship import DirtyShipCrawler
-from .discourse import DISCOURSE_CRAWLERS, DiscourseCrawler
-from .doodstream import DoodStreamCrawler
-from .dropbox import DropboxCrawler
-from .e621 import E621Crawler
-from .efukt import EfuktCrawler
-from .ehentai import EHentaiCrawler
-from .eightmuses import EightMusesCrawler
-from .eporner import EpornerCrawler
-from .erome import EromeCrawler, EromeFanCrawler
-from .fapello import FapelloCrawler
-from .fileditch import FileditchCrawler
-from .files_vc import FilesVcCrawler
-from .filester import FilesterCrawler
-from .flickr import FlickrCrawler
-from .flugel_anime import FlugelAnimeCrawler
-from .fourchan import FourChanCrawler
-from .fsiblog import FSIBlogCrawler
-from .fucking_fast import FuckingFastCrawler
-from .fuxxx import FuXXXCrawler
-from .girlsreleased import GirlsReleasedCrawler
-from .gofile import GoFileCrawler
-from .google_drive import GoogleDriveCrawler
-from .google_photos import GooglePhotosCrawler
-from .gupload import GUploadCrawler
-from .hclips import HClipsCrawler
-from .hdzog import HDZogCrawler
-from .hianime import HiAnimeCrawler
-from .hitomi_la import HitomiLaCrawler
-from .hotleak_vip import HotLeakVipCrawler
-from .hotleaks_dot_tv import HotLeaksTVCrawler
-from .hotmovs import HotMovsCrawler
-from .hotpic import HotPicCrawler
-from .iceyfile import IceyFileCrawler
-from .imagebam import ImageBamCrawler
-from .imagepond import ImagePondCrawler
-from .imgbb import ImgBBCrawler
-from .imgbox import ImgBoxCrawler
-from .imglike import ImgLikeCrawler
-from .imgur import ImgurCrawler
-from .imx_to import ImxToCrawler
-from .incestflix import IncestflixCrawler
-from .influencer_bitches import InfluencerBitchesCrawler
-from .inporn import InPornCrawler
-from .invision import INVISION_CRAWLERS
-from .jpg5 import JPG5Crawler
-from .kemono import KemonoCrawler
-from .koofr import KooFrCrawler
-from .leakedzone import LeakedZoneCrawler
-from .luscious import LusciousCrawler
-from .luxuretv import LuxureTVCrawler
-from .masahub import MasahubCrawler
-from .mediafire import MediaFireCrawler
-from .mega_nz import MegaNzCrawler
-from .megacloud import MegaCloudCrawler
-from .missav import MissAVCrawler
-from .mixdrop import MixDropCrawler
-from .motherless import MotherlessCrawler
-from .mydesi import MyDesiCrawler
-from .nekohouse import NekohouseCrawler
-from .nhentai import NHentaiCrawler
-from .noodle_magazine import NoodleMagazineCrawler
-from .nsfw_xxx import NsfwXXXCrawler
-from .nudostartv import NudoStarTVCrawler
-from .odnoklassniki import OdnoklassnikiCrawler
-from .omegascans import OmegaScansCrawler
-from .onedrive import OneDriveCrawler
-from .pcloud import PCloudCrawler
-from .pimp_bunny import PimpBunnyCrawler
-from .pimpandhost import PimpAndHostCrawler
-from .pixeldrain import PixelDrainCrawler
-from .pixhost import PixHostCrawler
-from .pkmncards import PkmncardsCrawler
-from .pmvhaven import PMVHavenCrawler
-from .pornhub import PornHubCrawler
-from .pornpics import PornPicsCrawler
-from .porntrex import PorntrexCrawler
-from .pornzog import PornZogCrawler
-from .postimg import PostImgCrawler
-from .ranoz import RanozCrawler, RootzCrawler
-from .realbooru import RealBooruCrawler
-from .redgifs import RedGifsCrawler
-from .rule34vault import Rule34VaultCrawler
-from .rule34video import Rule34VideoCrawler
-from .rule34xxx import Rule34XXXCrawler
-from .rule34xyz import Rule34XYZCrawler
-from .rumble import RumbleCrawler
-from .safe_soul import SafeSoulCrawler
-from .scrolller import ScrolllerCrawler
-from .send_now import SendNowCrawler
-from .sendvid import SendVidCrawler
-from .sex_dot_com import SexDotComCrawler
-from .spankbang import SpankBangCrawler
-from .streamable import StreamableCrawler
-from .streamtape import StreamtapeCrawler
-from .thisvid import ThisVidCrawler
-from .thothub import ThotHubCrawler
-from .tiktok import TikTokCrawler
-from .tnaflix import TNAFlixCrawler
-from .tokyomotion import TokioMotionCrawler
-from .toonily import ToonilyCrawler
-from .tranny_one import TrannyOneCrawler
-from .transfer_it import TransferItCrawler
-from .transflix import TransflixCrawler
-from .tubepornclassic import TubePornClassicCrawler
-from .turbovid import TurboVidCrawler
-from .twitch import TwitchCrawler
-from .twitter import TwitterCrawler
-from .twitter_images import TwimgCrawler
-from .twpornstars import TwPornstarsCrawler
-from .txxx import TXXXCrawler
-from .upload_ee import UploadEECrawler
-from .upornia import UPorniaCrawler
-from .vbulletin import VBULLETIN_CRAWLERS
-from .vidara import VidaraCrawler
-from .vipr_dot_im import ViprImCrawler
-from .vjav import VJavCrawler
-from .voe_sx import VoeSxCrawler
-from .voyeurhit import VoyeurHitCrawler
-from .vsco import VSCOCrawler
-from .vxxx import VXXXCrawler
-from .wetransfer import WeTransferCrawler
-from .wordpress import WP_CRAWLERS, WordPressHTMLCrawler, WordPressMediaCrawler
-from .xasiat import XasiatCrawler
-from .xbunkr import XBunkrCrawler
-from .xenforo import XF_CRAWLERS, SimpCityCrawler
-from .xgroovy import XGroovyCrawler
-from .xhamster import XhamsterCrawler
-from .xmilf import XMilfCrawler
-from .xvideos import XVideosCrawler
-from .xxxbunker import XXXBunkerCrawler
-from .yandex_disk import YandexDiskCrawler
-from .youjizz import YouJizzCrawler
-
-FORUM_CRAWLERS = XF_CRAWLERS.union(INVISION_CRAWLERS, DISCOURSE_CRAWLERS, VBULLETIN_CRAWLERS)
-GENERIC_CRAWLERS: set[type[Crawler]] = {WordPressHTMLCrawler, WordPressMediaCrawler, DiscourseCrawler, CheveretoCrawler}
-ALL_CRAWLERS: set[type[Crawler]] = {
- crawler for name, crawler in globals().items() if name.endswith("Crawler") and crawler is not Crawler
-}
-ALL_CRAWLERS.update(WP_CRAWLERS, GENERIC_CRAWLERS, FORUM_CRAWLERS)
-DEBUG_CRAWLERS = {SimpCityCrawler, TwitterCrawler}
-CRAWLERS = ALL_CRAWLERS
-WEBSITE_CRAWLERS = CRAWLERS - FORUM_CRAWLERS
-__all__ = ["ALL_CRAWLERS", "CRAWLERS", "DEBUG_CRAWLERS", "FORUM_CRAWLERS", "Crawler"]
diff --git a/cyberdrop_dl/licenses/LICENSE-APACHE b/cyberdrop_dl/licenses/LICENSE-APACHE
deleted file mode 100644
index d65719f80..000000000
--- a/cyberdrop_dl/licenses/LICENSE-APACHE
+++ /dev/null
@@ -1,201 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "{}"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright 2019 richardARPANET
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index a18f35b92..4d361a3d4 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -6,7 +6,7 @@ import datetime
import logging
import re
from pathlib import Path
-from typing import TYPE_CHECKING, Literal, Self
+from typing import TYPE_CHECKING, Literal, Self, TypeVar
import aiofiles
from yarl import URL
@@ -21,7 +21,6 @@ from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawler
from cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler, WordPressMediaCrawler
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItem
from cyberdrop_dl.exceptions import JDownloaderError, NoExtensionError
-from cyberdrop_dl.managers.manager import Manager
from cyberdrop_dl.utils.logger import log_spacer
from cyberdrop_dl.utils.utilities import get_download_path, remove_trailing_slash
@@ -31,10 +30,12 @@ if TYPE_CHECKING:
import aiosqlite
from cyberdrop_dl.config.global_model import GenericCrawlerInstances, GlobalSettings
+ from cyberdrop_dl.managers.manager import Manager
-
+_T = TypeVar("_T")
+_CrawlerT = TypeVar("_CrawlerT", bound=Crawler)
logger = logging.getLogger(__name__)
-existing_crawlers: dict[str, Crawler] = {}
+existing_crawlers: dict[str, type[Crawler]] = {}
_seen_urls: set[AbsoluteHttpURL] = set()
_crawlers_disabled_at_runtime: set[str] = set()
@@ -82,11 +83,16 @@ class ScrapeMapper:
"""Starts all scrapers."""
from cyberdrop_dl import plugins
- self.existing_crawlers = get_crawlers_mapping(self.manager)
+ crawlers = get_crawlers_mapping()
+
generic_crawlers = create_generic_crawlers_by_config(self.global_settings.generic_crawlers_instances)
for crawler in generic_crawlers:
- register_crawler(self.existing_crawlers, crawler(self.manager), from_user=True)
- disable_crawlers_by_config(self.existing_crawlers, self.global_settings.general.disable_crawlers)
+ register_crawler(crawlers, crawler, from_user=True)
+
+ disable_crawlers_by_config(crawlers, self.global_settings.general.disable_crawlers)
+
+ self.existing_crawlers = {domain: crawler(self.manager) for domain, crawler in crawlers.items()}
+
plugins.load(self.manager)
async def start_real_debrid(self) -> None:
@@ -358,27 +364,34 @@ def _create_item_from_row(row: aiosqlite.Row) -> ScrapeItem:
return item
-def get_crawlers_mapping(manager: Manager | None = None, include_generics: bool = False) -> dict[str, Crawler]:
+def get_crawlers_mapping(include_generics: bool = False) -> dict[str, type[Crawler]]:
"""Returns a mapping with an instance of all crawlers.
Crawlers are only created on the first calls. Future calls always return a reference to the same crawlers
If manager is `None`, the `MOCK_MANAGER` will be used, which means the crawlers won't be able to actually run"""
- from cyberdrop_dl.crawlers import CRAWLERS
+ from cyberdrop_dl.crawlers.crawler import Registry
- manager_ = manager or Manager()
global existing_crawlers
- if not existing_crawlers:
- for crawler in CRAWLERS:
- crawler_instance = crawler(manager_)
- register_crawler(existing_crawlers, crawler_instance, include_generics)
+ if existing_crawlers:
+ return existing_crawlers
+
+ Registry.import_all()
+ crawlers = Registry.generic | Registry.concrete
+
+ for crawler in crawlers:
+ register_crawler(existing_crawlers, crawler, include_generics)
+
+ copy = existing_crawlers.copy()
+ existing_crawlers.clear()
+ existing_crawlers.update(sorted(copy.items()))
return existing_crawlers
def register_crawler(
- existing_crawlers: dict[str, Crawler],
- crawler: Crawler,
+ existing_crawlers: dict[str, type[Crawler]],
+ crawler: type[Crawler],
include_generics: bool = False,
from_user: bool | Literal["raise"] = False,
) -> None:
@@ -411,10 +424,6 @@ def register_crawler(
existing_crawlers[domain] = crawler
-def get_unique_crawlers() -> list[Crawler]:
- return sorted(set(get_crawlers_mapping(include_generics=True).values()), key=lambda x: x.INFO.site)
-
-
def create_generic_crawlers_by_config(generic_crawlers: GenericCrawlerInstances) -> set[type[Crawler]]:
new_crawlers: set[type[Crawler]] = set()
if generic_crawlers.wordpress_html:
@@ -428,7 +437,7 @@ def create_generic_crawlers_by_config(generic_crawlers: GenericCrawlerInstances)
return new_crawlers
-def disable_crawlers_by_config(existing_crawlers: dict[str, Crawler], crawlers_to_disable: list[str]) -> None:
+def disable_crawlers_by_config(existing_crawlers: dict[str, type[Crawler]], crawlers_to_disable: list[str]) -> None:
if not crawlers_to_disable:
return
@@ -451,13 +460,14 @@ def disable_crawlers_by_config(existing_crawlers: dict[str, Crawler], crawlers_t
existing_crawlers.clear()
existing_crawlers.update(new_crawlers_mapping)
crawlers_info = "\n".join(
- str({info.site: info.supported_domains}) for info in sorted(crawlers.INFO for crawlers in disabled_crawlers)
+ str({info.site: info.supported_domains}) for info in sorted(c.INFO for c in disabled_crawlers)
)
logger.info(f"Crawlers disabled by config: \n{crawlers_info}")
+
log_spacer(10)
-def match_url_to_crawler(existing_crawlers: dict[str, Crawler], url: AbsoluteHttpURL) -> Crawler | None:
+def match_url_to_crawler(existing_crawlers: dict[str, _T], url: AbsoluteHttpURL) -> _T | None:
# match exact domain
if crawler := existing_crawlers.get(url.host):
return crawler
diff --git a/tests/test_scrape_mapper.py b/tests/test_scrape_mapper.py
index 71523b318..7236b175d 100644
--- a/tests/test_scrape_mapper.py
+++ b/tests/test_scrape_mapper.py
@@ -1,18 +1,11 @@
from __future__ import annotations
-from typing import TYPE_CHECKING
-
import pytest
-from cyberdrop_dl import crawlers
+from cyberdrop_dl.crawlers._chevereto import CheveretoCrawler
from cyberdrop_dl.crawlers.crawler import create_crawlers
from cyberdrop_dl.scraper import scrape_mapper
-if TYPE_CHECKING:
- from cyberdrop_dl.managers.manager import Manager
-
-TEST_BASE_CRAWLER = next(iter(crawlers.GENERIC_CRAWLERS))
-
@pytest.mark.parametrize(
"link",
@@ -21,17 +14,16 @@ TEST_BASE_CRAWLER = next(iter(crawlers.GENERIC_CRAWLERS))
"https://dropbox.uk",
"https://forum.allporncomix.com",
"https://cyberfile.me/abhl",
- "https://forums.plex.tv/"
+ "https://forums.plex.tv/",
"https://forums.socialmediagirls.com/threads/en-fr-tools-to-download-upload-content-websites-softwares-extensions.13930/#post-2070848",
+ "https://forums.plex.tv/",
],
)
-def test_generic_crawlers_that_match_supported_crawlers_should_not_be_created(
- running_manager: Manager, link: str
-) -> None:
- _ = scrape_mapper.get_crawlers_mapping(running_manager)
- crawler = next(iter(create_crawlers([link], TEST_BASE_CRAWLER)))
+def test_generic_crawlers_that_match_supported_crawlers_should_not_be_created(link: str) -> None:
+ crawlers = scrape_mapper.get_crawlers_mapping().copy()
+ crawler = next(iter(create_crawlers([link], CheveretoCrawler)))
with pytest.raises(ValueError) as exc_info:
- scrape_mapper.register_crawler(scrape_mapper.existing_crawlers, crawler(running_manager), from_user="raise")
+ scrape_mapper.register_crawler(crawlers, crawler, from_user="raise")
assert f"Unable to assign {link.split('/')[0]}" in str(exc_info.value)
@@ -43,15 +35,13 @@ def test_generic_crawlers_that_match_supported_crawlers_should_not_be_created(
"https://meta.discourse.org/",
],
)
-def test_generic_crawlers_that_do_no_match_supported_crawlers_should_be_created(
- running_manager: Manager, link: str
-) -> None:
- _ = scrape_mapper.get_crawlers_mapping(running_manager)
- crawler = next(iter(create_crawlers([link], TEST_BASE_CRAWLER)))
+def test_generic_crawlers_that_do_no_match_supported_crawlers_should_be_created(link: str) -> None:
+ _ = scrape_mapper.get_crawlers_mapping()
+ crawler = next(iter(create_crawlers([link], CheveretoCrawler)))
existing_crawlers = scrape_mapper.existing_crawlers.copy()
crawlers_before = set(existing_crawlers.values())
- scrape_mapper.register_crawler(existing_crawlers, crawler(running_manager), from_user="raise")
+ scrape_mapper.register_crawler(existing_crawlers, crawler, from_user="raise")
new_crawlers = set(existing_crawlers.values()) - crawlers_before
assert len(new_crawlers) == 1
created_crawler = next(iter(new_crawlers))
- assert issubclass(type(created_crawler), TEST_BASE_CRAWLER)
+ assert issubclass(created_crawler, CheveretoCrawler)