Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: use crawler registry for everything (#86)
NTFSvolumee86eb2f09042
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex a34babaea..e69de29bb 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -1,169 +0,0 @@-# ruff: noqa: F401-from __future__ import annotations--from cyberdrop_dl import env--from ._chevereto import CheveretoCrawler-from .anontransfer import AnonTransferCrawler-from .anysex import AnySexCrawler-from .archivebate import ArchiveBateCrawler-from .ashemaletube import AShemaleTubeCrawler-from .bandcamp import BandcampCrawler-from .beeg import BeegComCrawler-from .box_dot_com import BoxDotComCrawler-from .bunkrr import BunkrrCrawler-from .bunkrr_albums_io import BunkrAlbumsIOCrawler-from .buzzheavier import BuzzHeavierCrawler-from .camwhores_dot_tv import CamwhoresTVCrawler-from .catbox import CatboxCrawler-from .cloud_mail_ru import CloudMailRuCrawler-from .cloudflare_stream import CloudflareStreamCrawler-from .coomer import CoomerCrawler-from .crawler import Crawler-from .cyberdrop import CyberdropCrawler-from .cyberfile import CyberfileCrawler-from .desivideo import DesiVideoCrawler-from .dirtyship import DirtyShipCrawler-from .discourse import DISCOURSE_CRAWLERS, DiscourseCrawler-from .doodstream import DoodStreamCrawler-from .dropbox import DropboxCrawler-from .e621 import E621Crawler-from .efukt import EfuktCrawler-from .ehentai import EHentaiCrawler-from .eightmuses import EightMusesCrawler-from .eporner import EpornerCrawler-from .erome import EromeCrawler, EromeFanCrawler-from .fapello import FapelloCrawler-from .fileditch import FileditchCrawler-from .files_vc import FilesVcCrawler-from .filester import FilesterCrawler-from .flickr import FlickrCrawler-from .flugel_anime import FlugelAnimeCrawler-from .fourchan import FourChanCrawler-from .fsiblog import FSIBlogCrawler-from .fucking_fast import FuckingFastCrawler-from .fuxxx import FuXXXCrawler-from .girlsreleased import GirlsReleasedCrawler-from .gofile import GoFileCrawler-from .google_drive import GoogleDriveCrawler-from .google_photos import GooglePhotosCrawler-from .gupload import GUploadCrawler-from .hclips import HClipsCrawler-from .hdzog import HDZogCrawler-from .hianime import HiAnimeCrawler-from .hitomi_la import HitomiLaCrawler-from .hotleak_vip import HotLeakVipCrawler-from .hotleaks_dot_tv import HotLeaksTVCrawler-from .hotmovs import HotMovsCrawler-from .hotpic import HotPicCrawler-from .iceyfile import IceyFileCrawler-from .imagebam import ImageBamCrawler-from .imagepond import ImagePondCrawler-from .imgbb import ImgBBCrawler-from .imgbox import ImgBoxCrawler-from .imglike import ImgLikeCrawler-from .imgur import ImgurCrawler-from .imx_to import ImxToCrawler-from .incestflix import IncestflixCrawler-from .influencer_bitches import InfluencerBitchesCrawler-from .inporn import InPornCrawler-from .invision import INVISION_CRAWLERS-from .jpg5 import JPG5Crawler-from .kemono import KemonoCrawler-from .koofr import KooFrCrawler-from .leakedzone import LeakedZoneCrawler-from .luscious import LusciousCrawler-from .luxuretv import LuxureTVCrawler-from .masahub import MasahubCrawler-from .mediafire import MediaFireCrawler-from .mega_nz import MegaNzCrawler-from .megacloud import MegaCloudCrawler-from .missav import MissAVCrawler-from .mixdrop import MixDropCrawler-from .motherless import MotherlessCrawler-from .mydesi import MyDesiCrawler-from .nekohouse import NekohouseCrawler-from .nhentai import NHentaiCrawler-from .noodle_magazine import NoodleMagazineCrawler-from .nsfw_xxx import NsfwXXXCrawler-from .nudostartv import NudoStarTVCrawler-from .odnoklassniki import OdnoklassnikiCrawler-from .omegascans import OmegaScansCrawler-from .onedrive import OneDriveCrawler-from .pcloud import PCloudCrawler-from .pimp_bunny import PimpBunnyCrawler-from .pimpandhost import PimpAndHostCrawler-from .pixeldrain import PixelDrainCrawler-from .pixhost import PixHostCrawler-from .pkmncards import PkmncardsCrawler-from .pmvhaven import PMVHavenCrawler-from .pornhub import PornHubCrawler-from .pornpics import PornPicsCrawler-from .porntrex import PorntrexCrawler-from .pornzog import PornZogCrawler-from .postimg import PostImgCrawler-from .ranoz import RanozCrawler, RootzCrawler-from .realbooru import RealBooruCrawler-from .redgifs import RedGifsCrawler-from .rule34vault import Rule34VaultCrawler-from .rule34video import Rule34VideoCrawler-from .rule34xxx import Rule34XXXCrawler-from .rule34xyz import Rule34XYZCrawler-from .rumble import RumbleCrawler-from .safe_soul import SafeSoulCrawler-from .scrolller import ScrolllerCrawler-from .send_now import SendNowCrawler-from .sendvid import SendVidCrawler-from .sex_dot_com import SexDotComCrawler-from .spankbang import SpankBangCrawler-from .streamable import StreamableCrawler-from .streamtape import StreamtapeCrawler-from .thisvid import ThisVidCrawler-from .thothub import ThotHubCrawler-from .tiktok import TikTokCrawler-from .tnaflix import TNAFlixCrawler-from .tokyomotion import TokioMotionCrawler-from .toonily import ToonilyCrawler-from .tranny_one import TrannyOneCrawler-from .transfer_it import TransferItCrawler-from .transflix import TransflixCrawler-from .tubepornclassic import TubePornClassicCrawler-from .turbovid import TurboVidCrawler-from .twitch import TwitchCrawler-from .twitter import TwitterCrawler-from .twitter_images import TwimgCrawler-from .twpornstars import TwPornstarsCrawler-from .txxx import TXXXCrawler-from .upload_ee import UploadEECrawler-from .upornia import UPorniaCrawler-from .vbulletin import VBULLETIN_CRAWLERS-from .vidara import VidaraCrawler-from .vipr_dot_im import ViprImCrawler-from .vjav import VJavCrawler-from .voe_sx import VoeSxCrawler-from .voyeurhit import VoyeurHitCrawler-from .vsco import VSCOCrawler-from .vxxx import VXXXCrawler-from .wetransfer import WeTransferCrawler-from .wordpress import WP_CRAWLERS, WordPressHTMLCrawler, WordPressMediaCrawler-from .xasiat import XasiatCrawler-from .xbunkr import XBunkrCrawler-from .xenforo import XF_CRAWLERS, SimpCityCrawler-from .xgroovy import XGroovyCrawler-from .xhamster import XhamsterCrawler-from .xmilf import XMilfCrawler-from .xvideos import XVideosCrawler-from .xxxbunker import XXXBunkerCrawler-from .yandex_disk import YandexDiskCrawler-from .youjizz import YouJizzCrawler--FORUM_CRAWLERS = XF_CRAWLERS.union(INVISION_CRAWLERS, DISCOURSE_CRAWLERS, VBULLETIN_CRAWLERS)-GENERIC_CRAWLERS: set[type[Crawler]] = {WordPressHTMLCrawler, WordPressMediaCrawler, DiscourseCrawler, CheveretoCrawler}-ALL_CRAWLERS: set[type[Crawler]] = {- crawler for name, crawler in globals().items() if name.endswith("Crawler") and crawler is not Crawler-}-ALL_CRAWLERS.update(WP_CRAWLERS, GENERIC_CRAWLERS, FORUM_CRAWLERS)-DEBUG_CRAWLERS = {SimpCityCrawler, TwitterCrawler}-CRAWLERS = ALL_CRAWLERS-WEBSITE_CRAWLERS = CRAWLERS - FORUM_CRAWLERS-__all__ = ["ALL_CRAWLERS", "CRAWLERS", "DEBUG_CRAWLERS", "FORUM_CRAWLERS", "Crawler"]diff --git a/cyberdrop_dl/licenses/LICENSE-APACHE b/cyberdrop_dl/licenses/LICENSE-APACHEdeleted file mode 100644index d65719f80..000000000--- a/cyberdrop_dl/licenses/LICENSE-APACHE+++ /dev/null@@ -1,201 +0,0 @@- Apache License- Version 2.0, January 2004- http://www.apache.org/licenses/-- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION-- 1. Definitions.-- "License" shall mean the terms and conditions for use, reproduction,- and distribution as defined by Sections 1 through 9 of this document.-- "Licensor" shall mean the copyright owner or entity authorized by- the copyright owner that is granting the License.-- "Legal Entity" shall mean the union of the acting entity and all- other entities that control, are controlled by, or are under common- control with that entity. For the purposes of this definition,- "control" means (i) the power, direct or indirect, to cause the- direction or management of such entity, whether by contract or- otherwise, or (ii) ownership of fifty percent (50%) or more of the- outstanding shares, or (iii) beneficial ownership of such entity.-- "You" (or "Your") shall mean an individual or Legal Entity- exercising permissions granted by this License.-- "Source" form shall mean the preferred form for making modifications,- including but not limited to software source code, documentation- source, and configuration files.-- "Object" form shall mean any form resulting from mechanical- transformation or translation of a Source form, including but- not limited to compiled object code, generated documentation,- and conversions to other media types.-- "Work" shall mean the work of authorship, whether in Source or- Object form, made available under the License, as indicated by a- copyright notice that is included in or attached to the work- (an example is provided in the Appendix below).-- "Derivative Works" shall mean any work, whether in Source or Object- form, that is based on (or derived from) the Work and for which the- editorial revisions, annotations, elaborations, or other modifications- represent, as a whole, an original work of authorship. For the purposes- of this License, Derivative Works shall not include works that remain- separable from, or merely link (or bind by name) to the interfaces of,- the Work and Derivative Works thereof.-- "Contribution" shall mean any work of authorship, including- the original version of the Work and any modifications or additions- to that Work or Derivative Works thereof, that is intentionally- submitted to Licensor for inclusion in the Work by the copyright owner- or by an individual or Legal Entity authorized to submit on behalf of- the copyright owner. For the purposes of this definition, "submitted"- means any form of electronic, verbal, or written communication sent- to the Licensor or its representatives, including but not limited to- communication on electronic mailing lists, source code control systems,- and issue tracking systems that are managed by, or on behalf of, the- Licensor for the purpose of discussing and improving the Work, but- excluding communication that is conspicuously marked or otherwise- designated in writing by the copyright owner as "Not a Contribution."-- "Contributor" shall mean Licensor and any individual or Legal Entity- on behalf of whom a Contribution has been received by Licensor and- subsequently incorporated within the Work.-- 2. Grant of Copyright License. Subject to the terms and conditions of- this License, each Contributor hereby grants to You a perpetual,- worldwide, non-exclusive, no-charge, royalty-free, irrevocable- copyright license to reproduce, prepare Derivative Works of,- publicly display, publicly perform, sublicense, and distribute the- Work and such Derivative Works in Source or Object form.-- 3. Grant of Patent License. Subject to the terms and conditions of- this License, each Contributor hereby grants to You a perpetual,- worldwide, non-exclusive, no-charge, royalty-free, irrevocable- (except as stated in this section) patent license to make, have made,- use, offer to sell, sell, import, and otherwise transfer the Work,- where such license applies only to those patent claims licensable- by such Contributor that are necessarily infringed by their- Contribution(s) alone or by combination of their Contribution(s)- with the Work to which such Contribution(s) was submitted. If You- institute patent litigation against any entity (including a- cross-claim or counterclaim in a lawsuit) alleging that the Work- or a Contribution incorporated within the Work constitutes direct- or contributory patent infringement, then any patent licenses- granted to You under this License for that Work shall terminate- as of the date such litigation is filed.-- 4. Redistribution. You may reproduce and distribute copies of the- Work or Derivative Works thereof in any medium, with or without- modifications, and in Source or Object form, provided that You- meet the following conditions:-- (a) You must give any other recipients of the Work or- Derivative Works a copy of this License; and-- (b) You must cause any modified files to carry prominent notices- stating that You changed the files; and-- (c) You must retain, in the Source form of any Derivative Works- that You distribute, all copyright, patent, trademark, and- attribution notices from the Source form of the Work,- excluding those notices that do not pertain to any part of- the Derivative Works; and-- (d) If the Work includes a "NOTICE" text file as part of its- distribution, then any Derivative Works that You distribute must- include a readable copy of the attribution notices contained- within such NOTICE file, excluding those notices that do not- pertain to any part of the Derivative Works, in at least one- of the following places: within a NOTICE text file distributed- as part of the Derivative Works; within the Source form or- documentation, if provided along with the Derivative Works; or,- within a display generated by the Derivative Works, if and- wherever such third-party notices normally appear. The contents- of the NOTICE file are for informational purposes only and- do not modify the License. You may add Your own attribution- notices within Derivative Works that You distribute, alongside- or as an addendum to the NOTICE text from the Work, provided- that such additional attribution notices cannot be construed- as modifying the License.-- You may add Your own copyright statement to Your modifications and- may provide additional or different license terms and conditions- for use, reproduction, or distribution of Your modifications, or- for any such Derivative Works as a whole, provided Your use,- reproduction, and distribution of the Work otherwise complies with- the conditions stated in this License.-- 5. Submission of Contributions. Unless You explicitly state otherwise,- any Contribution intentionally submitted for inclusion in the Work- by You to the Licensor shall be under the terms and conditions of- this License, without any additional terms or conditions.- Notwithstanding the above, nothing herein shall supersede or modify- the terms of any separate license agreement you may have executed- with Licensor regarding such Contributions.-- 6. Trademarks. This License does not grant permission to use the trade- names, trademarks, service marks, or product names of the Licensor,- except as required for reasonable and customary use in describing the- origin of the Work and reproducing the content of the NOTICE file.-- 7. Disclaimer of Warranty. Unless required by applicable law or- agreed to in writing, Licensor provides the Work (and each- Contributor provides its Contributions) on an "AS IS" BASIS,- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or- implied, including, without limitation, any warranties or conditions- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A- PARTICULAR PURPOSE. You are solely responsible for determining the- appropriateness of using or redistributing the Work and assume any- risks associated with Your exercise of permissions under this License.-- 8. Limitation of Liability. In no event and under no legal theory,- whether in tort (including negligence), contract, or otherwise,- unless required by applicable law (such as deliberate and grossly- negligent acts) or agreed to in writing, shall any Contributor be- liable to You for damages, including any direct, indirect, special,- incidental, or consequential damages of any character arising as a- result of this License or out of the use or inability to use the- Work (including but not limited to damages for loss of goodwill,- work stoppage, computer failure or malfunction, or any and all- other commercial damages or losses), even if such Contributor- has been advised of the possibility of such damages.-- 9. Accepting Warranty or Additional Liability. While redistributing- the Work or Derivative Works thereof, You may choose to offer,- and charge a fee for, acceptance of support, warranty, indemnity,- or other liability obligations and/or rights consistent with this- License. However, in accepting such obligations, You may act only- on Your own behalf and on Your sole responsibility, not on behalf- of any other Contributor, and only if You agree to indemnify,- defend, and hold each Contributor harmless for any liability- incurred by, or claims asserted against, such Contributor by reason- of your accepting any such warranty or additional liability.-- END OF TERMS AND CONDITIONS-- APPENDIX: How to apply the Apache License to your work.-- To apply the Apache License to your work, attach the following- boilerplate notice, with the fields enclosed by brackets "{}"- replaced with your own identifying information. (Don't include- the brackets!) The text should be enclosed in the appropriate- comment syntax for the file format. We also recommend that a- file or class name and description of purpose be included on the- same "printed page" as the copyright notice for easier- identification within third-party archives.-- Copyright 2019 richardARPANET-- Licensed under the Apache License, Version 2.0 (the "License");- you may not use this file except in compliance with the License.- You may obtain a copy of the License at-- http://www.apache.org/licenses/LICENSE-2.0-- Unless required by applicable law or agreed to in writing, software- distributed under the License is distributed on an "AS IS" BASIS,- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- See the License for the specific language governing permissions and- limitations under the License.diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex a18f35b92..4d361a3d4 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -6,7 +6,7 @@ import datetimeimport loggingimport refrom pathlib import Path-from typing import TYPE_CHECKING, Literal, Self+from typing import TYPE_CHECKING, Literal, Self, TypeVarimport aiofilesfrom yarl import URL@@ -21,7 +21,6 @@ from cyberdrop_dl.crawlers.realdebrid import RealDebridCrawlerfrom cyberdrop_dl.crawlers.wordpress import WordPressHTMLCrawler, WordPressMediaCrawlerfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, ScrapeItemfrom cyberdrop_dl.exceptions import JDownloaderError, NoExtensionError-from cyberdrop_dl.managers.manager import Managerfrom cyberdrop_dl.utils.logger import log_spacerfrom cyberdrop_dl.utils.utilities import get_download_path, remove_trailing_slash@@ -31,10 +30,12 @@ if TYPE_CHECKING:import aiosqlitefrom cyberdrop_dl.config.global_model import GenericCrawlerInstances, GlobalSettings+ from cyberdrop_dl.managers.manager import Manager-+_T = TypeVar("_T")+_CrawlerT = TypeVar("_CrawlerT", bound=Crawler)logger = logging.getLogger(__name__)-existing_crawlers: dict[str, Crawler] = {}+existing_crawlers: dict[str, type[Crawler]] = {}_seen_urls: set[AbsoluteHttpURL] = set()_crawlers_disabled_at_runtime: set[str] = set()@@ -82,11 +83,16 @@ class ScrapeMapper:"""Starts all scrapers."""from cyberdrop_dl import plugins- self.existing_crawlers = get_crawlers_mapping(self.manager)+ crawlers = get_crawlers_mapping()+generic_crawlers = create_generic_crawlers_by_config(self.global_settings.generic_crawlers_instances)for crawler in generic_crawlers:- register_crawler(self.existing_crawlers, crawler(self.manager), from_user=True)- disable_crawlers_by_config(self.existing_crawlers, self.global_settings.general.disable_crawlers)+ register_crawler(crawlers, crawler, from_user=True)++ disable_crawlers_by_config(crawlers, self.global_settings.general.disable_crawlers)++ self.existing_crawlers = {domain: crawler(self.manager) for domain, crawler in crawlers.items()}+plugins.load(self.manager)async def start_real_debrid(self) -> None:@@ -358,27 +364,34 @@ def _create_item_from_row(row: aiosqlite.Row) -> ScrapeItem:return item-def get_crawlers_mapping(manager: Manager | None = None, include_generics: bool = False) -> dict[str, Crawler]:+def get_crawlers_mapping(include_generics: bool = False) -> dict[str, type[Crawler]]:"""Returns a mapping with an instance of all crawlers.Crawlers are only created on the first calls. Future calls always return a reference to the same crawlersIf manager is `None`, the `MOCK_MANAGER` will be used, which means the crawlers won't be able to actually run"""- from cyberdrop_dl.crawlers import CRAWLERS+ from cyberdrop_dl.crawlers.crawler import Registry- manager_ = manager or Manager()global existing_crawlers- if not existing_crawlers:- for crawler in CRAWLERS:- crawler_instance = crawler(manager_)- register_crawler(existing_crawlers, crawler_instance, include_generics)+ if existing_crawlers:+ return existing_crawlers++ Registry.import_all()+ crawlers = Registry.generic | Registry.concrete++ for crawler in crawlers:+ register_crawler(existing_crawlers, crawler, include_generics)++ copy = existing_crawlers.copy()+ existing_crawlers.clear()+ existing_crawlers.update(sorted(copy.items()))return existing_crawlersdef register_crawler(- existing_crawlers: dict[str, Crawler],- crawler: Crawler,+ existing_crawlers: dict[str, type[Crawler]],+ crawler: type[Crawler],include_generics: bool = False,from_user: bool | Literal["raise"] = False,) -> None:@@ -411,10 +424,6 @@ def register_crawler(existing_crawlers[domain] = crawler-def get_unique_crawlers() -> list[Crawler]:- return sorted(set(get_crawlers_mapping(include_generics=True).values()), key=lambda x: x.INFO.site)--def create_generic_crawlers_by_config(generic_crawlers: GenericCrawlerInstances) -> set[type[Crawler]]:new_crawlers: set[type[Crawler]] = set()if generic_crawlers.wordpress_html:@@ -428,7 +437,7 @@ def create_generic_crawlers_by_config(generic_crawlers: GenericCrawlerInstances)return new_crawlers-def disable_crawlers_by_config(existing_crawlers: dict[str, Crawler], crawlers_to_disable: list[str]) -> None:+def disable_crawlers_by_config(existing_crawlers: dict[str, type[Crawler]], crawlers_to_disable: list[str]) -> None:if not crawlers_to_disable:return@@ -451,13 +460,14 @@ def disable_crawlers_by_config(existing_crawlers: dict[str, Crawler], crawlers_texisting_crawlers.clear()existing_crawlers.update(new_crawlers_mapping)crawlers_info = "\n".join(- str({info.site: info.supported_domains}) for info in sorted(crawlers.INFO for crawlers in disabled_crawlers)+ str({info.site: info.supported_domains}) for info in sorted(c.INFO for c in disabled_crawlers))logger.info(f"Crawlers disabled by config: \n{crawlers_info}")+log_spacer(10)-def match_url_to_crawler(existing_crawlers: dict[str, Crawler], url: AbsoluteHttpURL) -> Crawler | None:+def match_url_to_crawler(existing_crawlers: dict[str, _T], url: AbsoluteHttpURL) -> _T | None:# match exact domainif crawler := existing_crawlers.get(url.host):return crawlerdiff --git a/tests/test_scrape_mapper.py b/tests/test_scrape_mapper.pyindex 71523b318..7236b175d 100644--- a/tests/test_scrape_mapper.py+++ b/tests/test_scrape_mapper.py@@ -1,18 +1,11 @@from __future__ import annotations-from typing import TYPE_CHECKING-import pytest-from cyberdrop_dl import crawlers+from cyberdrop_dl.crawlers._chevereto import CheveretoCrawlerfrom cyberdrop_dl.crawlers.crawler import create_crawlersfrom cyberdrop_dl.scraper import scrape_mapper-if TYPE_CHECKING:- from cyberdrop_dl.managers.manager import Manager--TEST_BASE_CRAWLER = next(iter(crawlers.GENERIC_CRAWLERS))-@pytest.mark.parametrize("link",@@ -21,17 +14,16 @@ TEST_BASE_CRAWLER = next(iter(crawlers.GENERIC_CRAWLERS))"https://dropbox.uk","https://forum.allporncomix.com","https://cyberfile.me/abhl",- "https://forums.plex.tv/"+ "https://forums.plex.tv/","https://forums.socialmediagirls.com/threads/en-fr-tools-to-download-upload-content-websites-softwares-extensions.13930/#post-2070848",+ "https://forums.plex.tv/",],)-def test_generic_crawlers_that_match_supported_crawlers_should_not_be_created(- running_manager: Manager, link: str-) -> None:- _ = scrape_mapper.get_crawlers_mapping(running_manager)- crawler = next(iter(create_crawlers([link], TEST_BASE_CRAWLER)))+def test_generic_crawlers_that_match_supported_crawlers_should_not_be_created(link: str) -> None:+ crawlers = scrape_mapper.get_crawlers_mapping().copy()+ crawler = next(iter(create_crawlers([link], CheveretoCrawler)))with pytest.raises(ValueError) as exc_info:- scrape_mapper.register_crawler(scrape_mapper.existing_crawlers, crawler(running_manager), from_user="raise")+ scrape_mapper.register_crawler(crawlers, crawler, from_user="raise")assert f"Unable to assign {link.split('/')[0]}" in str(exc_info.value)@@ -43,15 +35,13 @@ def test_generic_crawlers_that_match_supported_crawlers_should_not_be_created("https://meta.discourse.org/",],)-def test_generic_crawlers_that_do_no_match_supported_crawlers_should_be_created(- running_manager: Manager, link: str-) -> None:- _ = scrape_mapper.get_crawlers_mapping(running_manager)- crawler = next(iter(create_crawlers([link], TEST_BASE_CRAWLER)))+def test_generic_crawlers_that_do_no_match_supported_crawlers_should_be_created(link: str) -> None:+ _ = scrape_mapper.get_crawlers_mapping()+ crawler = next(iter(create_crawlers([link], CheveretoCrawler)))existing_crawlers = scrape_mapper.existing_crawlers.copy()crawlers_before = set(existing_crawlers.values())- scrape_mapper.register_crawler(existing_crawlers, crawler(running_manager), from_user="raise")+ scrape_mapper.register_crawler(existing_crawlers, crawler, from_user="raise")new_crawlers = set(existing_crawlers.values()) - crawlers_beforeassert len(new_crawlers) == 1created_crawler = next(iter(new_crawlers))- assert issubclass(type(created_crawler), TEST_BASE_CRAWLER)+ assert issubclass(created_crawler, CheveretoCrawler)