Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: use the same task_id for all subfolders in a fs (PixelDrain) (#1942)

NTFSvolume6f00e7cb4995
diff --git a/CHANGELOG.md b/CHANGELOG.md
index ac7f47cc5..6d15f3c4a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -40,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Fixed
- Some false positive DDoS-Guard detections
+- URLs extraction from text files in a filesystem (PixelDrain)
### Security
diff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.py
index 551674875..09579d4cf 100644
--- a/cyberdrop_dl/crawlers/pixeldrain.py
+++ b/cyberdrop_dl/crawlers/pixeldrain.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import asyncio
import dataclasses
from typing import TYPE_CHECKING, Any, ClassVar, Literal, final
@@ -234,17 +235,18 @@ class PixelDrainCrawler(Crawler):
subfolders = node.path.split("/")[2:-1]
new_scrape_item.append_folders(*subfolders)
- self.create_task(self._file_task(new_scrape_item, node))
+ tg.create_task(self._file_task(new_scrape_item, node))
elif node.type == "dir":
- self.create_task(subfolder(new_scrape_item, node.path))
+ tg.create_task(subfolder(new_scrape_item, node.path))
else:
self.raise_exc(new_scrape_item, f"Unknown node type: {node.type}")
scrape_item.add_children()
- walk_filesystem(fs)
+ async with asyncio.TaskGroup() as tg:
+ walk_filesystem(fs)
@error_handling_wrapper
async def file(self, scrape_item: ScrapeItem, file_id: str) -> None:
@@ -255,10 +257,12 @@ class PixelDrainCrawler(Crawler):
await self._file(scrape_item, file)
async def _file(self, scrape_item: ScrapeItem, file: File | Node) -> None:
+ src = _build_download_url(file).with_host(self.origin.host)
if "text/plain" in file.mime_type:
- return await self.text(scrape_item, file)
+ scrape_item.setup_as_album(self.create_title(file.name, file.id))
+ text = await self.api.text(src)
+ return self._text(scrape_item, text)
- src = _build_download_url(file).with_host(self.origin.host)
if await self.check_complete_by_hash(src, "sha256", file.hash_sha256):
return None
@@ -266,12 +270,7 @@ class PixelDrainCrawler(Crawler):
scrape_item.uploaded_at = self.parse_iso_date(file.date_upload)
await self.handle_file(src, scrape_item, file.name, ext, custom_filename=filename)
- @error_handling_wrapper
- async def text(self, scrape_item: ScrapeItem, file: File | Node) -> None:
- assert file.id
- scrape_item.setup_as_album(self.create_title(file.name, file.id))
- text = await self.api.text(file.id)
-
+ def _text(self, scrape_item: ScrapeItem, text: str) -> None:
for line in text.splitlines():
try:
link = self.parse_url(line)
@@ -296,24 +295,20 @@ class PixelDrainAPI(API):
async def file(self, file_id: str) -> File:
api_url = self.origin / "api/file" / file_id / "info"
- resp = await self._request(api_url)
+ resp = await self.text(api_url)
return type_adapter(File).validate_json(resp)
- async def text(self, file_id: str) -> str:
- api_url = self.origin / "api/file" / file_id
- return await self._request(api_url)
-
async def folder(self, list_id: str) -> Folder:
api_url = self.origin / "api/list" / list_id
- resp = await self._request(api_url)
+ resp = await self.text(api_url)
return type_adapter(Folder).validate_json(resp)
async def filesystem(self, path: str) -> FileSystem:
api_url = (self.origin / "api/filesystem" / path.removeprefix("/")).with_query("stat")
- resp = await self._request(api_url)
+ resp = await self.text(api_url)
return type_adapter(FileSystem).validate_json(resp)
- async def _request(self, api_url: AbsoluteHttpURL) -> str:
+ async def text(self, api_url: AbsoluteHttpURL) -> str:
return await self.request_text(api_url, headers=self.headers)