Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: release download lock while hashing

NTFSvolume4e9957e24501
diff --git a/cyberdrop_dl/clients/downloads.py b/cyberdrop_dl/clients/downloads.py
index 188e8d9b2..acb10b4f4 100644
--- a/cyberdrop_dl/clients/downloads.py
+++ b/cyberdrop_dl/clients/downloads.py
@@ -187,23 +187,8 @@ class DownloadClient:
downloaded = await self._download(domain, media_item)
if downloaded:
await aio.move(media_item.partial_file, media_item.path)
- if not media_item.is_segment:
- if await self.__skip_by_duration(media_item):
- return False
- await self.process_completed(media_item, domain)
- await self.handle_media_item_completion(media_item, downloaded=True)
return downloaded
- async def __skip_by_duration(self, media_item: MediaItem) -> bool:
- proceed = not await filter_by_duration(media_item, self.config)
- await self.manager.database.history.add_duration(media_item.domain, media_item)
- if not proceed:
- logger.info(f"Download skipped {media_item.url} due to runtime restrictions")
- await aio.unlink(media_item.path)
- await self.mark_incomplete(media_item, media_item.domain)
- self.manager.scrape_mapper.tui.files.stats.skipped += 1
- return not proceed
-
async def mark_incomplete(self, media_item: MediaItem, domain: str) -> None:
if media_item.is_segment:
return
diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.py
index a93f753ca..05b65ed09 100644
--- a/cyberdrop_dl/downloader/http.py
+++ b/cyberdrop_dl/downloader/http.py
@@ -256,7 +256,26 @@ class Downloader:
return False
async with self.__download_context(media_item):
- return await self._download(media_item)
+ downloaded = await self._download(media_item)
+
+ if not downloaded:
+ return False
+ assert not media_item.is_segment
+ if await self.__skip_by_duration(media_item):
+ return False
+ await self.client.process_completed(media_item, media_item.domain)
+ await self.client.handle_media_item_completion(media_item, downloaded=True)
+ return True
+
+ async def __skip_by_duration(self, media_item: MediaItem) -> bool:
+ proceed = not await filter_by_duration(media_item, self.config)
+ await self.manager.database.history.add_duration(media_item.domain, media_item)
+ if not proceed:
+ logger.info(f"Download skipped {media_item.url} due to runtime restrictions")
+ await aio.unlink(media_item.path)
+ await self.client.mark_incomplete(media_item, media_item.domain)
+ self.manager.scrape_mapper.tui.files.stats.skipped += 1
+ return not proceed
@error_handling_wrapper
async def download_hls(self, media_item: MediaItem, m3u8_group: Rendition) -> None: