Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
perf: skip duplicate URLs from within the same post (kemono) (#1306)
NTFSvolumef0bc85d7c420
diff --git a/cyberdrop_dl/crawlers/kemono.py b/cyberdrop_dl/crawlers/kemono.pyindex de5e01d51..e78401404 100644--- a/cyberdrop_dl/crawlers/kemono.py+++ b/cyberdrop_dl/crawlers/kemono.py@@ -399,14 +399,14 @@ class KemonoBaseCrawler(Crawler, is_abc=True):yield urldef __handle_post(self, scrape_item: ScrapeItem, post: Post) -> None:- for file in post.all_files:- file_url = self.__make_file_url(file)- self.create_task(self.handle_direct_link(scrape_item, file_url))- scrape_item.add_children()+ files = (self.__make_file_url(file) for file in post.all_files)- for file_url in post.soup_attachments:- self.create_task(self.handle_direct_link(scrape_item, file_url))- scrape_item.add_children()+ seen: set[AbsoluteHttpURL] = set()+ for url in itertools.chain(files, post.soup_attachments):+ if url not in seen:+ seen.add(url)+ self.create_task(self.handle_direct_link(scrape_item, url))+ scrape_item.add_children()self._handle_post_content(scrape_item, post)