Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: all tweets being skipped (twitter)
NTFSvolumecedd2cd1fc60
diff --git a/cyberdrop_dl/crawlers/twitter/crawler.py b/cyberdrop_dl/crawlers/twitter/crawler.pyindex 86133ee09..85a845c0e 100644--- a/cyberdrop_dl/crawlers/twitter/crawler.py+++ b/cyberdrop_dl/crawlers/twitter/crawler.py@@ -300,21 +300,21 @@ class TweetsFilterer:__repr__ = simple_repr("since", "until", "oldest_tweet")- def should_skip(self, tweet: Tweet) -> tuple[bool, str]:+ def should_skip(self, tweet: Tweet) -> str | bool:if tweet.id in self.seen:- return True, "duplicate"+ return "duplicate"self.seen.add(tweet.id)if self.oldest_tweet is None or self.oldest_tweet.created_timestamp > tweet.created_timestamp:self.oldest_tweet = tweetif self.until and tweet.created_timestamp > self.until:- return True, "too new"+ return "too new"if self.since and tweet.created_timestamp < self.since:- return True, "too old"+ return "too old"- return True, ""+ return Falseasync def __call__(self, tweet_pages: AsyncGenerator[Iterable[Tweet]]) -> AsyncGenerator[Tweet]:max_old_tweets = 10@@ -322,10 +322,9 @@ class TweetsFilterer:async for tweets in pages:old_tweets = 0for tweet in tweets:- skip, reason = self.should_skip(tweet)- if skip:- self.log.warning("Skipping tweet %s (%s)", tweet.id, reason)- if reason == "too old":+ if skip_reason := self.should_skip(tweet):+ self.log.warning("Skipping tweet %s (%s)", tweet.id, skip_reason)+ if skip_reason == "too old":old_tweets += 1continueyield tweet