Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: reduce public API surface
NTFSvolume89fe0fc0face
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 7cd64b2a0..f71c101e5 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -760,14 +760,13 @@ class Crawler(HTTPMixin, HLSMixin, ABC):attribute: str = "href",*,results: Mapping[str, bool] | None = None,- **kwargs: Any,) -> Generator[tuple[AbsoluteHttpURL | None, ScrapeItem]]:"""Generates tuples with an URL from the `src` value of the first image tag (AKA the thumbnail) and a new scrape item from the value of `attribute`:param results: must be the output of `self.get_album_results`.If provided, it will be used as a filter, to only yield items that has not been downloaded before"""for thumb, link in self.iter_tags(soup, selector, attribute, results=results):- new_scrape_item = scrape_item.create_child(link, **kwargs)+ new_scrape_item = scrape_item.create_child(link)yield thumb, new_scrape_itemscrape_item.add_children()diff --git a/cyberdrop_dl/crawlers/eightmuses.py b/cyberdrop_dl/crawlers/eightmuses.pyindex c400d94fa..b2f904c14 100644--- a/cyberdrop_dl/crawlers/eightmuses.py+++ b/cyberdrop_dl/crawlers/eightmuses.py@@ -50,7 +50,9 @@ class EightMusesCrawler(Crawler):image = css.select(tile, IMAGE_SELECTOR)is_new_album = image["itemtype"] == "https://schema.org/ImageGallery"new_album_id = f"{scrape_item.album_id}/{tile_title.replace(' ', '-')}"- new_scrape_item = scrape_item.create_child(tile_link, new_title_part=tile_title, album_id=new_album_id)+ new_scrape_item = scrape_item.create_child(tile_link)+ new_scrape_item.append_folder(tile_title)+ new_scrape_item.album_id = new_album_idif is_new_album:await self.album(new_scrape_item)continuediff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.pyindex 071b7393a..e35d9d48f 100644--- a/cyberdrop_dl/crawlers/eporner.py+++ b/cyberdrop_dl/crawlers/eporner.py@@ -115,7 +115,8 @@ class EpornerCrawler(Crawler):part, selector = partsurl = canonical_url / partasync for soup in self.web_pager(url):- for _, new_scrape_item in self.iter_children(scrape_item, soup, selector, new_title_part=name):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, selector):+ new_scrape_item.append_folder(name)self.create_task(self.run(new_scrape_item))@error_handling_wrapperdiff --git a/cyberdrop_dl/crawlers/motherless.py b/cyberdrop_dl/crawlers/motherless.pyindex 70cfac4ac..45094f211 100644--- a/cyberdrop_dl/crawlers/motherless.py+++ b/cyberdrop_dl/crawlers/motherless.py@@ -81,13 +81,15 @@ class MotherlessCrawler(Crawler):if is_homepage or "images" in scrape_item.url.parts:async for soup in self.web_pager(images_url):check_soup(soup)- for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR, new_title_part="Images"):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):+ new_scrape_item.append_folder("Images")self.create_task(self.run(new_scrape_item))if is_homepage or "videos" in scrape_item.url.parts:async for soup in self.web_pager(videos_url):check_soup(soup)- for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR, new_title_part="Videos"):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):+ new_scrape_item.append_folder("Videos")self.create_task(self.run(new_scrape_item))@error_handling_wrapper@@ -128,7 +130,8 @@ class MotherlessCrawler(Crawler):title = self.create_title(title, collection_id)scrape_item.setup_as_album(title, album_id=collection_id)- for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR, new_title_part=name):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):+ new_scrape_item.append_folder(name)self.create_task(self.run(new_scrape_item))@error_handling_wrapperdiff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.pyindex b3828599a..3836841e8 100644--- a/cyberdrop_dl/crawlers/porntrex.py+++ b/cyberdrop_dl/crawlers/porntrex.py@@ -136,7 +136,8 @@ class PorntrexCrawler(Crawler):albums_url = scrape_item.url / "albums"soup = await self.request_soup(albums_url)- for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ALBUMS, new_title_part="albums"):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ALBUMS):+ new_scrape_item.append_folder("albums")self.create_task(self.run(new_scrape_item))async def _ajax_pagination(self, scrape_item: ScrapeItem, last_page: int, **kwargs: str) -> None: # noqa: C901diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.pyindex e1c9fe16d..592593bee 100644--- a/cyberdrop_dl/crawlers/xhamster.py+++ b/cyberdrop_dl/crawlers/xhamster.py@@ -160,7 +160,8 @@ class XhamsterCrawler(Crawler):self, scrape_item: ScrapeItem, url: AbsoluteHttpURL, selector: str, name: str) -> None:async for soup in self.web_pager(url):- for _, new_scrape_item in self.iter_children(scrape_item, soup, selector, new_title_part=name):+ for _, new_scrape_item in self.iter_children(scrape_item, soup, selector):+ new_scrape_item.append_folder(name)self.create_task(self.run(new_scrape_item))@error_handling_wrapperdiff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.pyindex 31e3257dc..dfa2caffa 100644--- a/cyberdrop_dl/crawlers/xvideos.py+++ b/cyberdrop_dl/crawlers/xvideos.py@@ -150,7 +150,8 @@ class XVideosCrawler(Crawler):for gallery_id in galleries:url = scrape_item.url / part / gallery_id.removeprefix("f-")- new_scrape_item = scrape_item.create_child(url, new_title_part="photos")+ new_scrape_item = scrape_item.create_child(url)+ new_scrape_item.append_folder("photos")self.create_task(self.run(new_scrape_item))scrape_item.add_children()@@ -227,7 +228,8 @@ class XVideosCrawler(Crawler):url = scrape_item.url.origin() / f"video.{video['eid']}" / slugelse:url = self.parse_url(video["url"], scrape_item.url.origin())- new_scrape_item = scrape_item.create_child(url, new_title_part=new_part)+ new_scrape_item = scrape_item.create_child(url)+ new_scrape_item.append_folder(new_part)self.create_task(self.run(new_scrape_item))scrape_item.add_children()diff --git a/cyberdrop_dl/url_objects.py b/cyberdrop_dl/url_objects.pyindex 98175277a..5ebd99456 100644--- a/cyberdrop_dl/url_objects.py+++ b/cyberdrop_dl/url_objects.py@@ -302,7 +302,6 @@ class ScrapeItem:self,url: AbsoluteHttpURL,*,- new_title_part: str = "",part_of_album: bool = False,album_id: str | None = None,add_parent: AbsoluteHttpURL | bool | None = None,@@ -318,22 +317,13 @@ class ScrapeItem:assert is_absolute_http_url(new_parent)scrape_item.parents.append(new_parent)- if new_title_part:- scrape_item.append_folder(new_title_part)-scrape_item.url = urlscrape_item.part_of_album = part_of_album or scrape_item.part_of_albumscrape_item.album_id = album_id or scrape_item.album_idreturn scrape_item- def create_child(self, url: AbsoluteHttpURL, *, new_title_part: str = "", album_id: str | None = None) -> Self:- return self.create_new(- url,- part_of_album=True,- add_parent=True,- new_title_part=new_title_part,- album_id=album_id,- )+ def create_child(self, url: AbsoluteHttpURL) -> Self:+ return self.create_new(url, part_of_album=True, add_parent=True)def setup_as_album(self: ScrapeItem, title: str, *, album_id: str | None = None) -> None:return self.setup_as(title, FILE_HOST_ALBUM, album_id=album_id)