Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: better date filter (twitter)
NTFSvolumedd5982148ccb
diff --git a/cyberdrop_dl/crawlers/twitter/api.py b/cyberdrop_dl/crawlers/twitter/api.pyindex feda1db2e..867f2cbff 100644--- a/cyberdrop_dl/crawlers/twitter/api.py+++ b/cyberdrop_dl/crawlers/twitter/api.py@@ -2,15 +2,17 @@# https://github.com/FxEmbed/FxEmbedfrom __future__ import annotations+import dataclasses+import itertoolsfrom contextvars import ContextVar-from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Unpack+from typing import TYPE_CHECKING, Any, ClassVar, Literal, Unpackfrom cyberdrop_dl import envfrom cyberdrop_dl.cache import disk_cached_methodfrom cyberdrop_dl.clients.http import HTTPConfigfrom cyberdrop_dl.crawlers.crawler import APIfrom cyberdrop_dl.crawlers.twitter.models import Broadcast, Tweet-from cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.exceptions import DownloadError, ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLif TYPE_CHECKING:@@ -23,8 +25,8 @@ if TYPE_CHECKING:@HTTPConfig(rate_limit=(3, 1)) # Actual limit is 1000 req/min (~ 16.7 req/s) per IPclass FXTwitterAPI(API):ENTRYPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://api.fxtwitter.com/2")- cursor: ClassVar[ContextVar[str | None]] = ContextVar("cursor")since: ClassVar[ContextVar[int | None]] = ContextVar("since")+ until: ClassVar[ContextVar[int | None]] = ContextVar("until")def __post_init__(self) -> None:self.user: UserEndpoint = UserEndpoint(self)@@ -41,57 +43,67 @@ class FXTwitterAPI(API):tweets = (t for t in resp["thread"] if t["type"] == "status")return map(Tweet.model_validate, tweets)- @classmethod- def build_gql_search_query(- cls,- query: str | None = None,- *,- user_screen_name: str | None = None,- max_tweet_id: str | None = None,- include_retweets: bool = False,- include_text_only: bool = False,- ) -> str:- def parts() -> Generator[str]:- if user_screen_name:- yield f"from:{user_screen_name}"- if max_tweet_id:- yield f"max_id:{max_tweet_id}"- if include_retweets:- yield "include:nativeretweets"- yield "include:retweets"- if not include_text_only:- yield "filter:links"- if query:- yield query-- return " ".join(parts())-def search(self, query: str, feed: Literal["latest", "top", "media"] = "latest") -> AsyncGenerator[map[Tweet]]:- url = (self.ENTRYPOINT / "search").with_query(q=query, feed=feed)+ url = self.prepare_search(query, feed=feed)return self.pager(url)- async def pager(self, url: AbsoluteHttpURL) -> AsyncGenerator[map[Tweet]]:- url = url.update_query(count=100)- if cursor := self.cursor.get():- url = url.update_query(cursor=cursor)- if since := self.since.get():- url = url.update_query(since=since)+ @classmethod+ def prepare_search(+ cls,+ query: str,+ *,+ gql_filter: TwitterGQLSearchFilter | None = None,+ feed: Literal["latest", "top", "media"] = "latest",+ ) -> AbsoluteHttpURL:+ if gql_filter is not None:+ query = " ".join(filter(None, (query, *gql_filter)))++ return (cls.ENTRYPOINT / "search").with_query(q=query, feed=feed)++ async def pager(self, base_url: AbsoluteHttpURL, cursor: str | None = None) -> AsyncGenerator[map[Tweet]]:+ url = base_url.update_query(+ {+ name: value+ for name, value in (+ ("count", 100),+ ("cursor", cursor),+ ("since", self.since.get()),+ ("until", self.until.get()),+ )+ if value+ }+ )empty_pages: int = 0- while True:- resp = await self.request_json(url)+ max_empty_pages: int = env.TWITTER_MAX_EMPTY_PAGES++ for page in itertools.count(1):+ self.log.debug("Fetching page #%s (%s)", page, base_url)+ try:+ resp = await self.request_json(url)+ except DownloadError as e:+ if e.status == 404 and "search" in url.parts:+ raise DownloadError(403, "Search results blocked by twitter") from None+ raiseif resp["results"]:- empty_pages = 0+ n_results = len(resp["results"])+ if empty_pages:+ empty_pages = 0+ self.log.info(+ "Found %s results for %s on page %s after %s empty pages", n_results, url, page, empty_pages+ )yield map(Tweet.model_validate, resp["results"])else:empty_pages += 1+cursor = resp["cursor"].get("bottom")- self.cursor.set(cursor)if not cursor or url.query.get("cursor") == cursor:break- if empty_pages >= env.TWITTER_MAX_EMPTY_PAGES:- self.log.warning("Stopping pagination on %s after %s empty responses", url, empty_pages)++ if empty_pages > max_empty_pages:+ self.log.warning("Stopping pagination on %s after too many empty responses (%s)", url, empty_pages)break+url = url.update_query(cursor=cursor)@@ -105,6 +117,26 @@ class UserEndpoint(API.Endpoint[FXTwitterAPI]):return self.api.pager(url)[email protected](frozen=True, slots=True, kw_only=True)+class TwitterGQLSearchFilter:+ user: str | None = None+ since: int | None = None+ until: int | None = None+ max_id: str | None = None+ include: tuple[str, ...] = ()+ filter: tuple[str, ...] = ()++ def __iter__(self) -> Generator[str]:+ for field in dataclasses.fields(self):+ name, value = field.name, getattr(self, field.name)+ if name == "user":+ name = "from"+ values = value if type(value) is tuple else (value,)+ for val in values:+ if val:+ yield f"{name}:{val}"++_DEFAULT_CRED = ("AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA")@@ -153,8 +185,6 @@ class TwitterAPI(API):class BroadcastEndpoint(API.Endpoint[TwitterAPI]):- ENDED: Final = "ENDED"-async def __call__(self, broadcast_id: str) -> Broadcast:url = (self.api.ENTRYPOINT / "broadcasts/show.json").with_query(ids=broadcast_id)resp = await self.api.request_json(url)@@ -169,8 +199,9 @@ class BroadcastEndpoint(API.Endpoint[TwitterAPI]):def _parse(self, bd: dict[str, Any]) -> Broadcast:if not bd:raise ScrapeError(404)- state: str = bd.setdefault("state", self.ENDED)- if state != self.ENDED:++ state = bd.get("state", "ENDED")+ if state != "ENDED":raise ScrapeError(422, f"{state} broadcasts are not supported")return Broadcast.model_validate(bd)diff --git a/cyberdrop_dl/crawlers/twitter/crawler.py b/cyberdrop_dl/crawlers/twitter/crawler.pyindex e30c24f5e..567fef754 100644--- a/cyberdrop_dl/crawlers/twitter/crawler.py+++ b/cyberdrop_dl/crawlers/twitter/crawler.py@@ -6,16 +6,15 @@ import datetimefrom pathlib import Pathfrom typing import TYPE_CHECKING, ClassVar, override-from cyberdrop_dl import aiofrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths-from cyberdrop_dl.crawlers.twitter.api import FXTwitterAPI, TwitterAPI+from cyberdrop_dl.crawlers.twitter.api import FXTwitterAPI, TwitterAPI, TwitterGQLSearchFilterfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.mediaprops import Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItemfrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:- from collections.abc import AsyncGenerator, AsyncIterable, Generator, Iterable+ from collections.abc import AsyncGenerator, Generator, Iterablefrom cyberdrop_dl.config.crawlers import TwitterConfigfrom cyberdrop_dl.crawlers.twitter.models import Broadcast, Tweet@@ -109,6 +108,7 @@ class TwitterCrawler(Crawler):DOMAIN: ClassVar[str] = "twitter"DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date:%Y-%m-%d} - {id}"_default_since: int | None = None+ _default_until: int | None = Nonedef __post_init__(self) -> None:self.__config__: TwitterConfig = self.config.crawlers.twitter@@ -116,6 +116,8 @@ class TwitterCrawler(Crawler):self.x_api: TwitterAPI = TwitterAPI.from_crawler(self)if after := self.config.filters.after:self._default_since = int(datetime.datetime.combine(after, datetime.time.min).timestamp())+ if before := self.config.filters.before:+ self._default_until = int(datetime.datetime.combine(before, datetime.time.min).timestamp())@classmethod@override@@ -133,8 +135,7 @@ class TwitterCrawler(Crawler):case [_, "status", status_id, *_] | ["i", "web", "status", status_id]:fn = self.thread if self.__config__.threads else self.tweetawait fn(scrape_item, status_id)- case [user, "media"]:- await self.user(scrape_item, user, include_retweets=False)+case ["i", "broadcasts", bd_id]:await self.broadcast(scrape_item, bd_id)case ["i", "events", event_id]:@@ -142,8 +143,10 @@ class TwitterCrawler(Crawler):case ["search"] if query := query_get("q"):feed = query_get("f") or query_get("feed")await self.search(scrape_item, query, feed)+ case [user, "media"]:+ await self.user(scrape_item, user, media_only=True)case [user]:- await self.user(scrape_item, user, include_retweets=self.__config__.retweets)+ await self.user(scrape_item, user, media_only=False)case _:raise ValueError@@ -182,52 +185,71 @@ class TwitterCrawler(Crawler):scrape_item.add_children()@error_handling_wrapper- async def user(self, scrape_item: ScrapeItem, user: str, *, include_retweets: bool) -> None:+ async def user(self, scrape_item: ScrapeItem, user: str, *, media_only: bool) -> None:# A single endpoint does not provide all user tweets# We need to call all endpoints and then searchscrape_item.setup_as_profile("")- filter_tweets = TweetsDeduper()- pages = self.api.user.media(user)-- if include_retweets:- pages = aio.chain(pages, self.api.user.tweets(user))-- await self._iter_tweets(scrape_item, filter_tweets.pages(pages))- gql_query = self.api.build_gql_search_query(- user_screen_name=user,- max_tweet_id=filter_tweets.last_tweet,- include_retweets=include_retweets,- include_text_only=include_retweets,+ tweets_filter = self._prepare_filter(scrape_item.url)+ await self._iter_tweets(scrape_item, tweets_filter(self.api.user.media(user)))+ await self._iter_tweets(scrape_item, tweets_filter(self.api.user.tweets(user)))++ oldest = tweets_filter.oldest_tweet+ if not oldest:+ return++ tweets_filter.until = (+ min(tweets_filter.until, oldest.created_timestamp) if tweets_filter.until else oldest.created_timestamp)- with self.catch_errors(scrape_item.url.with_fragment("posts_search_results")):- self.log.info("Starting manual search query for posts of %s", user)- await self._iter_tweets(scrape_item, filter_tweets.pages(self.api.search(gql_query)))+ self.api.until.set(tweets_filter.until)+ feed = "media" if media_only else "top"+ api_url = self.api.prepare_search(+ query="",+ feed=feed,+ gql_filter=TwitterGQLSearchFilter(+ user=user,+ max_id=oldest.id,+ until=tweets_filter.until,+ since=tweets_filter.since,+ include=("nativeretweets", "retweets") if self.__config__.retweets else (),+ filter=("links",) if media_only else (),+ ),+ )++ with self.catch_errors((self.PRIMARY_URL / "search").with_query(api_url.query)):+ self.log.info(+ "Starting manual search for posts of %s (feed: '%s', raw_query: '%s')",+ user,+ feed,+ api_url.query["q"],+ )+ await self._iter_tweets(scrape_item, tweets_filter(self.api.pager(api_url)))@error_handling_wrapperasync def search(self, scrape_item: ScrapeItem, query: str, feed: str | None = None) -> None:scrape_item.setup_as_forum("")feed = feed if feed in {"latest", "top", "media"} else "latest"- await self._iter_tweets(scrape_item, self.api.search(query, feed)) # pyright: ignore[reportArgumentType]-- async def _iter_tweets(self, scrape_item: ScrapeItem, tweets_pages: AsyncIterable[Iterable[Tweet]]) -> None:- with self._cursor_ctx(scrape_item.url):- async for tweets in tweets_pages:- for tweet in tweets:- if tweet.reposted_by and not self.__config__.retweets:- self.log.warning(- "skipping tweet %s by config options [retweet]. Original author: @%s, retweeted by: @%s",- tweet.id,- tweet.author["screen_name"],- tweet.reposted_by["screen_name"],- )- continue- new_item = scrape_item.create_child(self.parse_url(tweet.url))- self._tweet(new_item, tweet)- scrape_item.add_children()-- def _since(self, url: AbsoluteHttpURL) -> int | None:- for query_name in ("since", "after"):+ tweets_filter = self._prepare_filter(scrape_item.url)+ await self._iter_tweets(scrape_item, tweets_filter(self.api.search(query, feed))) # pyright: ignore[reportArgumentType]++ async def _iter_tweets(self, scrape_item: ScrapeItem, tweets_pages: AsyncGenerator[Tweet]) -> None:+ async with contextlib.aclosing(tweets_pages) as tweets:+ async for tweet in tweets:+ if not self.__config__.retweets and tweet.reposted_by and tweet.author["id"] != tweet.reposted_by["id"]:+ self.log.warning(+ "skipping tweet %s by config options [retweet]. Original author: @%s, retweeted by: @%s",+ tweet.id,+ tweet.author["screen_name"],+ tweet.reposted_by["screen_name"],+ )+ continue++ new_item = scrape_item.create_child(self.parse_url(tweet.url))+ self._tweet(new_item, tweet)+ scrape_item.add_children()++ def _ts_from_query(self, url: AbsoluteHttpURL, *keys: str) -> int | None:+ for query_name in keys:if value := url.query.get(query_name):try:return int(value)@@ -237,24 +259,15 @@ class TwitterCrawler(Crawler):except ValueError:self.log.error("Unable to parse '%s' query param from %s, ignoring...", query_name, url)- return self._default_since-- @contextlib.contextmanager- def _cursor_ctx(self, url: AbsoluteHttpURL) -> Generator[None]:+ def _prepare_filter(self, url: AbsoluteHttpURL) -> TweetsFilterer:# TODO: make cursor endpoint aware- init_cursor = url.query.get("cursor")- self.api.cursor.set(init_cursor)- self.api.since.set(self._since(url))- try:- yield- finally:- cursor = self.api.cursor.get()- if cursor and cursor != init_cursor:- self.log.info(- "Use cursor '%s' as a query param to resume scraping from the same point. ex: %s",- cursor,- url.update_query(cursor=cursor),- )+ tweets_filter = TweetsFilterer(+ since=self._ts_from_query(url, "since", "after") or self._default_since,+ until=self._ts_from_query(url, "until", "before") or self._default_until,+ )+ self.api.since.set(tweets_filter.since)+ self.api.until.set(tweets_filter.until)+ return tweets_filter@error_handling_wrapperasync def broadcast(self, scrape_item: ScrapeItem, broadcast_id: str, *, event: bool = False) -> None:@@ -286,21 +299,41 @@ class TwitterCrawler(Crawler):@dataclasses.dataclass(slots=True)-class TweetsDeduper:- seen: set[str] = dataclasses.field(default_factory=set)- last_tweet: str | None = None+class TweetsFilterer:+ ABORT: ClassVar[str] = "ABORT"+ since: int | None = None+ until: int | None = None- def __call__(self, tweets: Iterable[Tweet]) -> Generator[Tweet]:- for tweet in tweets:- if tweet.id in self.seen:- continue- self.last_tweet = tweet.id- self.seen.add(tweet.id)- yield tweet+ seen: set[str] = dataclasses.field(init=False, default_factory=set)+ oldest_tweet: Tweet | None = dataclasses.field(init=False, default=None)++ def should_scrape(self, tweet: Tweet) -> bool:+ if tweet.id in self.seen:+ return False++ self.seen.add(tweet.id)+ if self.oldest_tweet is None or self.oldest_tweet.created_timestamp > tweet.created_timestamp:+ self.oldest_tweet = tweet++ if self.until and tweet.created_timestamp > self.until:+ return False- async def pages(self, pages: AsyncIterable[Iterable[Tweet]]) -> AsyncGenerator[Iterable[Tweet]]:- async for tweets in pages:- yield self(tweets)+ if self.since and tweet.created_timestamp < self.since:+ raise StopIteration++ return True++ async def __call__(self, tweet_pages: AsyncGenerator[Iterable[Tweet]]) -> AsyncGenerator[Tweet]:+ async with contextlib.aclosing(tweet_pages) as pages:+ async for tweets in pages:+ for tweet in tweets:+ try:+ scrape = self.should_scrape(tweet)+ except StopIteration:+ return+ else:+ if scrape:+ yield tweet@dataclasses.dataclass(slots=True)diff --git a/cyberdrop_dl/crawlers/twitter/models.py b/cyberdrop_dl/crawlers/twitter/models.pyindex 197bc319a..988200793 100644--- a/cyberdrop_dl/crawlers/twitter/models.py+++ b/cyberdrop_dl/crawlers/twitter/models.py@@ -188,7 +188,6 @@ class Broadcast(DeferredModel):user_display_name: strusername: strtwitter_username: str- state: stris_locked: boolheight: intwidth: intdiff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.pyindex 707c02a65..aba2b23f2 100644--- a/cyberdrop_dl/env.py+++ b/cyberdrop_dl/env.py@@ -58,7 +58,7 @@ TERMUX = {FILEDITCH_WAIT = int(_env("FILEDITCH_WAIT") or 20)GOFILE_SALT = _env("GOFILE_SALT")-TWITTER_MAX_EMPTY_PAGES = int(_env("TWITTER_MAX_EMPTY_PAGES") or 1)+TWITTER_MAX_EMPTY_PAGES = int(_env("TWITTER_MAX_EMPTY_PAGES") or 10)ALL_VARS = dict(sorted(ALL_VARS.items())) # pyright: ignore[reportConstantRedefinition]ALL_VARS_RESOLVED = dict(