Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: use the same task_id for all subfolders in a fs (PixelDrain) (#1942)
NTFSvolume6f00e7cb4995
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex ac7f47cc5..6d15f3c4a 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -40,6 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0### Fixed- Some false positive DDoS-Guard detections+- URLs extraction from text files in a filesystem (PixelDrain)### Securitydiff --git a/cyberdrop_dl/crawlers/pixeldrain.py b/cyberdrop_dl/crawlers/pixeldrain.pyindex 551674875..09579d4cf 100644--- a/cyberdrop_dl/crawlers/pixeldrain.py+++ b/cyberdrop_dl/crawlers/pixeldrain.py@@ -2,6 +2,7 @@from __future__ import annotations+import asyncioimport dataclassesfrom typing import TYPE_CHECKING, Any, ClassVar, Literal, final@@ -234,17 +235,18 @@ class PixelDrainCrawler(Crawler):subfolders = node.path.split("/")[2:-1]new_scrape_item.append_folders(*subfolders)- self.create_task(self._file_task(new_scrape_item, node))+ tg.create_task(self._file_task(new_scrape_item, node))elif node.type == "dir":- self.create_task(subfolder(new_scrape_item, node.path))+ tg.create_task(subfolder(new_scrape_item, node.path))else:self.raise_exc(new_scrape_item, f"Unknown node type: {node.type}")scrape_item.add_children()- walk_filesystem(fs)+ async with asyncio.TaskGroup() as tg:+ walk_filesystem(fs)@error_handling_wrapperasync def file(self, scrape_item: ScrapeItem, file_id: str) -> None:@@ -255,10 +257,12 @@ class PixelDrainCrawler(Crawler):await self._file(scrape_item, file)async def _file(self, scrape_item: ScrapeItem, file: File | Node) -> None:+ src = _build_download_url(file).with_host(self.origin.host)if "text/plain" in file.mime_type:- return await self.text(scrape_item, file)+ scrape_item.setup_as_album(self.create_title(file.name, file.id))+ text = await self.api.text(src)+ return self._text(scrape_item, text)- src = _build_download_url(file).with_host(self.origin.host)if await self.check_complete_by_hash(src, "sha256", file.hash_sha256):return None@@ -266,12 +270,7 @@ class PixelDrainCrawler(Crawler):scrape_item.uploaded_at = self.parse_iso_date(file.date_upload)await self.handle_file(src, scrape_item, file.name, ext, custom_filename=filename)- @error_handling_wrapper- async def text(self, scrape_item: ScrapeItem, file: File | Node) -> None:- assert file.id- scrape_item.setup_as_album(self.create_title(file.name, file.id))- text = await self.api.text(file.id)-+ def _text(self, scrape_item: ScrapeItem, text: str) -> None:for line in text.splitlines():try:link = self.parse_url(line)@@ -296,24 +295,20 @@ class PixelDrainAPI(API):async def file(self, file_id: str) -> File:api_url = self.origin / "api/file" / file_id / "info"- resp = await self._request(api_url)+ resp = await self.text(api_url)return type_adapter(File).validate_json(resp)- async def text(self, file_id: str) -> str:- api_url = self.origin / "api/file" / file_id- return await self._request(api_url)-async def folder(self, list_id: str) -> Folder:api_url = self.origin / "api/list" / list_id- resp = await self._request(api_url)+ resp = await self.text(api_url)return type_adapter(Folder).validate_json(resp)async def filesystem(self, path: str) -> FileSystem:api_url = (self.origin / "api/filesystem" / path.removeprefix("/")).with_query("stat")- resp = await self._request(api_url)+ resp = await self.text(api_url)return type_adapter(FileSystem).validate_json(resp)- async def _request(self, api_url: AbsoluteHttpURL) -> str:+ async def text(self, api_url: AbsoluteHttpURL) -> str:return await self.request_text(api_url, headers=self.headers)