Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: reduce public API surface

NTFSvolume89fe0fc0face
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 7cd64b2a0..f71c101e5 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -760,14 +760,13 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
attribute: str = "href",
*,
results: Mapping[str, bool] | None = None,
- **kwargs: Any,
) -> Generator[tuple[AbsoluteHttpURL | None, ScrapeItem]]:
"""Generates tuples with an URL from the `src` value of the first image tag (AKA the thumbnail) and a new scrape item from the value of `attribute`
:param results: must be the output of `self.get_album_results`.
If provided, it will be used as a filter, to only yield items that has not been downloaded before"""
for thumb, link in self.iter_tags(soup, selector, attribute, results=results):
- new_scrape_item = scrape_item.create_child(link, **kwargs)
+ new_scrape_item = scrape_item.create_child(link)
yield thumb, new_scrape_item
scrape_item.add_children()
diff --git a/cyberdrop_dl/crawlers/eightmuses.py b/cyberdrop_dl/crawlers/eightmuses.py
index c400d94fa..b2f904c14 100644
--- a/cyberdrop_dl/crawlers/eightmuses.py
+++ b/cyberdrop_dl/crawlers/eightmuses.py
@@ -50,7 +50,9 @@ class EightMusesCrawler(Crawler):
image = css.select(tile, IMAGE_SELECTOR)
is_new_album = image["itemtype"] == "https://schema.org/ImageGallery"
new_album_id = f"{scrape_item.album_id}/{tile_title.replace(' ', '-')}"
- new_scrape_item = scrape_item.create_child(tile_link, new_title_part=tile_title, album_id=new_album_id)
+ new_scrape_item = scrape_item.create_child(tile_link)
+ new_scrape_item.append_folder(tile_title)
+ new_scrape_item.album_id = new_album_id
if is_new_album:
await self.album(new_scrape_item)
continue
diff --git a/cyberdrop_dl/crawlers/eporner.py b/cyberdrop_dl/crawlers/eporner.py
index 071b7393a..e35d9d48f 100644
--- a/cyberdrop_dl/crawlers/eporner.py
+++ b/cyberdrop_dl/crawlers/eporner.py
@@ -115,7 +115,8 @@ class EpornerCrawler(Crawler):
part, selector = parts
url = canonical_url / part
async for soup in self.web_pager(url):
- for _, new_scrape_item in self.iter_children(scrape_item, soup, selector, new_title_part=name):
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, selector):
+ new_scrape_item.append_folder(name)
self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
diff --git a/cyberdrop_dl/crawlers/motherless.py b/cyberdrop_dl/crawlers/motherless.py
index 70cfac4ac..45094f211 100644
--- a/cyberdrop_dl/crawlers/motherless.py
+++ b/cyberdrop_dl/crawlers/motherless.py
@@ -81,13 +81,15 @@ class MotherlessCrawler(Crawler):
if is_homepage or "images" in scrape_item.url.parts:
async for soup in self.web_pager(images_url):
check_soup(soup)
- for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR, new_title_part="Images"):
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):
+ new_scrape_item.append_folder("Images")
self.create_task(self.run(new_scrape_item))
if is_homepage or "videos" in scrape_item.url.parts:
async for soup in self.web_pager(videos_url):
check_soup(soup)
- for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR, new_title_part="Videos"):
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):
+ new_scrape_item.append_folder("Videos")
self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
@@ -128,7 +130,8 @@ class MotherlessCrawler(Crawler):
title = self.create_title(title, collection_id)
scrape_item.setup_as_album(title, album_id=collection_id)
- for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR, new_title_part=name):
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, ITEM_SELECTOR):
+ new_scrape_item.append_folder(name)
self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
diff --git a/cyberdrop_dl/crawlers/porntrex.py b/cyberdrop_dl/crawlers/porntrex.py
index b3828599a..3836841e8 100644
--- a/cyberdrop_dl/crawlers/porntrex.py
+++ b/cyberdrop_dl/crawlers/porntrex.py
@@ -136,7 +136,8 @@ class PorntrexCrawler(Crawler):
albums_url = scrape_item.url / "albums"
soup = await self.request_soup(albums_url)
- for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ALBUMS, new_title_part="albums"):
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, Selector.ALBUMS):
+ new_scrape_item.append_folder("albums")
self.create_task(self.run(new_scrape_item))
async def _ajax_pagination(self, scrape_item: ScrapeItem, last_page: int, **kwargs: str) -> None: # noqa: C901
diff --git a/cyberdrop_dl/crawlers/xhamster.py b/cyberdrop_dl/crawlers/xhamster.py
index e1c9fe16d..592593bee 100644
--- a/cyberdrop_dl/crawlers/xhamster.py
+++ b/cyberdrop_dl/crawlers/xhamster.py
@@ -160,7 +160,8 @@ class XhamsterCrawler(Crawler):
self, scrape_item: ScrapeItem, url: AbsoluteHttpURL, selector: str, name: str
) -> None:
async for soup in self.web_pager(url):
- for _, new_scrape_item in self.iter_children(scrape_item, soup, selector, new_title_part=name):
+ for _, new_scrape_item in self.iter_children(scrape_item, soup, selector):
+ new_scrape_item.append_folder(name)
self.create_task(self.run(new_scrape_item))
@error_handling_wrapper
diff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.py
index 31e3257dc..dfa2caffa 100644
--- a/cyberdrop_dl/crawlers/xvideos.py
+++ b/cyberdrop_dl/crawlers/xvideos.py
@@ -150,7 +150,8 @@ class XVideosCrawler(Crawler):
for gallery_id in galleries:
url = scrape_item.url / part / gallery_id.removeprefix("f-")
- new_scrape_item = scrape_item.create_child(url, new_title_part="photos")
+ new_scrape_item = scrape_item.create_child(url)
+ new_scrape_item.append_folder("photos")
self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
@@ -227,7 +228,8 @@ class XVideosCrawler(Crawler):
url = scrape_item.url.origin() / f"video.{video['eid']}" / slug
else:
url = self.parse_url(video["url"], scrape_item.url.origin())
- new_scrape_item = scrape_item.create_child(url, new_title_part=new_part)
+ new_scrape_item = scrape_item.create_child(url)
+ new_scrape_item.append_folder(new_part)
self.create_task(self.run(new_scrape_item))
scrape_item.add_children()
diff --git a/cyberdrop_dl/url_objects.py b/cyberdrop_dl/url_objects.py
index 98175277a..5ebd99456 100644
--- a/cyberdrop_dl/url_objects.py
+++ b/cyberdrop_dl/url_objects.py
@@ -302,7 +302,6 @@ class ScrapeItem:
self,
url: AbsoluteHttpURL,
*,
- new_title_part: str = "",
part_of_album: bool = False,
album_id: str | None = None,
add_parent: AbsoluteHttpURL | bool | None = None,
@@ -318,22 +317,13 @@ class ScrapeItem:
assert is_absolute_http_url(new_parent)
scrape_item.parents.append(new_parent)
- if new_title_part:
- scrape_item.append_folder(new_title_part)
-
scrape_item.url = url
scrape_item.part_of_album = part_of_album or scrape_item.part_of_album
scrape_item.album_id = album_id or scrape_item.album_id
return scrape_item
- def create_child(self, url: AbsoluteHttpURL, *, new_title_part: str = "", album_id: str | None = None) -> Self:
- return self.create_new(
- url,
- part_of_album=True,
- add_parent=True,
- new_title_part=new_title_part,
- album_id=album_id,
- )
+ def create_child(self, url: AbsoluteHttpURL) -> Self:
+ return self.create_new(url, part_of_album=True, add_parent=True)
def setup_as_album(self: ScrapeItem, title: str, *, album_id: str | None = None) -> None:
return self.setup_as(title, FILE_HOST_ALBUM, album_id=album_id)