Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: all tweets being skipped (twitter)

NTFSvolumecedd2cd1fc60
diff --git a/cyberdrop_dl/crawlers/twitter/crawler.py b/cyberdrop_dl/crawlers/twitter/crawler.py
index 86133ee09..85a845c0e 100644
--- a/cyberdrop_dl/crawlers/twitter/crawler.py
+++ b/cyberdrop_dl/crawlers/twitter/crawler.py
@@ -300,21 +300,21 @@ class TweetsFilterer:
__repr__ = simple_repr("since", "until", "oldest_tweet")
- def should_skip(self, tweet: Tweet) -> tuple[bool, str]:
+ def should_skip(self, tweet: Tweet) -> str | bool:
if tweet.id in self.seen:
- return True, "duplicate"
+ return "duplicate"
self.seen.add(tweet.id)
if self.oldest_tweet is None or self.oldest_tweet.created_timestamp > tweet.created_timestamp:
self.oldest_tweet = tweet
if self.until and tweet.created_timestamp > self.until:
- return True, "too new"
+ return "too new"
if self.since and tweet.created_timestamp < self.since:
- return True, "too old"
+ return "too old"
- return True, ""
+ return False
async def __call__(self, tweet_pages: AsyncGenerator[Iterable[Tweet]]) -> AsyncGenerator[Tweet]:
max_old_tweets = 10
@@ -322,10 +322,9 @@ class TweetsFilterer:
async for tweets in pages:
old_tweets = 0
for tweet in tweets:
- skip, reason = self.should_skip(tweet)
- if skip:
- self.log.warning("Skipping tweet %s (%s)", tweet.id, reason)
- if reason == "too old":
+ if skip_reason := self.should_skip(tweet):
+ self.log.warning("Skipping tweet %s (%s)", tweet.id, skip_reason)
+ if skip_reason == "too old":
old_tweets += 1
continue
yield tweet