Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add twitter post support (#1633)
NTFSvolume03d1dc01cc18
diff --git a/cyberdrop_dl/constants.py b/cyberdrop_dl/constants.pyindex 0c2805149..04b75a74f 100644--- a/cyberdrop_dl/constants.py+++ b/cyberdrop_dl/constants.py@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Finalfrom aiohttp.resolver import AsyncResolver, ThreadedResolverfrom rich.text import Text+from cyberdrop_dl import envfrom cyberdrop_dl.compat import Enum, StrEnumif TYPE_CHECKING:@@ -63,7 +64,6 @@ class TempExt(StrEnum):class BlockedDomains:partial_match = ("facebook",- "twitter.com","instagram","fbcdn","gfycat",@@ -77,9 +77,13 @@ class BlockedDomains:"beacons.page","beacons.ai","allmylinks.com",- ".x.com",)- exact_match = ("x.com",)++ exact_match = ()++ if not env.ENABLE_TWITTER:+ partial_match = *partial_match, "twitter.com", ".x.com"+ exact_match = *exact_match, "x.com"DEFAULT_APP_STORAGE = Path("./AppData")diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex 6fcd75689..8cdd5ee7a 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -127,6 +127,7 @@ from .transflix import TransflixCrawlerfrom .tubepornclassic import TubePornClassicCrawlerfrom .turbovid import TurboVidCrawlerfrom .twitch import TwitchCrawler+from .twitter import TwitterCrawlerfrom .twitter_images import TwimgCrawlerfrom .twpornstars import TwPornstarsCrawlerfrom .txxx import TXXXCrawler@@ -156,7 +157,7 @@ ALL_CRAWLERS: set[type[Crawler]] = {crawler for name, crawler in globals().items() if name.endswith("Crawler") and crawler is not Crawler}ALL_CRAWLERS.update(WP_CRAWLERS, GENERIC_CRAWLERS, FORUM_CRAWLERS)-DEBUG_CRAWLERS = {SimpCityCrawler}+DEBUG_CRAWLERS = {SimpCityCrawler, TwitterCrawler}if env.ENABLE_DEBUG_CRAWLERS == "d396ab8c85fcb1fecd22c8d9b58acf944a44e6d35014e9dd39e42c9a64091eda":CRAWLERS = ALL_CRAWLERSelse:diff --git a/cyberdrop_dl/crawlers/twitter.py b/cyberdrop_dl/crawlers/twitter.pynew file mode 100644index 000000000..99c551166--- /dev/null+++ b/cyberdrop_dl/crawlers/twitter.py@@ -0,0 +1,55 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Any, ClassVar++from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils.utilities import error_handling_wrapper++if TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem+++_API_URL = AbsoluteHttpURL("https://api.fxtwitter.com")+++class TwitterCrawler(Crawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Tweet": "/<handle>/status/<tweet_id>",+ }+ OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("twitter.com",)+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://x.com")+ DOMAIN: ClassVar[str] = "twitter"+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date:%Y-%m-%d} - {id}"++ @property+ def separate_posts(self) -> bool:+ return True++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case [_user, "status", _]:+ return await self.tweet(scrape_item)+ case _:+ raise ValueError++ @error_handling_wrapper+ async def tweet(self, scrape_item: ScrapeItem) -> None:+ url = _API_URL / scrape_item.url.path.removeprefix("/")+ tweet: dict[str, Any] = (await self.request_json(url))["tweet"]+ scrape_item.possible_datetime = tweet["created_timestamp"]+ name = tweet["author"]["screen_name"]+ post_title = self.create_separate_post_title(None, tweet["id"], scrape_item.possible_datetime)+ scrape_item.setup_as_profile(self.create_title(f"@{name}"))+ scrape_item.add_to_parent_title(post_title)++ await self.write_metadata(scrape_item, tweet["id"], tweet)++ for media in tweet["media"]["all"]:+ if media["type"] == "video":+ media = max(media["formats"], key=lambda f: f.get("bitrate", 0))++ source = self.parse_url(media["url"])+ new_item = scrape_item.create_child(source)+ self.handle_external_links(new_item, reset=False)+ scrape_item.add_children()diff --git a/cyberdrop_dl/crawlers/twitter_images.py b/cyberdrop_dl/crawlers/twitter_images.pyindex cda752d6e..87ef71596 100644--- a/cyberdrop_dl/crawlers/twitter_images.py+++ b/cyberdrop_dl/crawlers/twitter_images.py@@ -43,6 +43,8 @@ class TwimgCrawler(Crawler):async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None = None) -> None:_, *media_item.fallbacks = list(_make_download_urls(media_item.url))+ if media_item.referer == media_item.url and media_item.parents:+ media_item.referer = media_item.parents[0]await super().handle_media_item(media_item, m3u8)diff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.pyindex bf8bb52ec..d2949d434 100644--- a/cyberdrop_dl/env.py+++ b/cyberdrop_dl/env.py@@ -22,3 +22,4 @@ NO_PLUGINS = bool(os.getenv("CDL_NO_PLUGINS"))PIXELDRAIN_PROXY = os.getenv("CDL_PIXELDRAIN_PROXY")BANDCAMP_FORMATS = os.getenv("CDL_BANDCAMP_FORMATS")+ENABLE_TWITTER = bool(os.getenv("CDL_ENABLE_TWITTER"))diff --git a/tests/crawlers/test_cases/twitter.py b/tests/crawlers/test_cases/twitter.pynew file mode 100644index 000000000..b6759dce9--- /dev/null+++ b/tests/crawlers/test_cases/twitter.py@@ -0,0 +1,18 @@+DOMAIN = "twitter"+TEST_CASES = [+ (+ "https://x.com/matthewisabel/status/2014444098690613718",+ [+ {+ "url": "https://video.twimg.com/amplify_video/2014443278259609602/vid/avc1/2744x2160/KfifaGUZoO2lsD79.mp4?tag=21",+ "filename": "KfifaGUZoO2lsD79.mp4",+ "debrid_link": None,+ "original_filename": "KfifaGUZoO2lsD79.mp4",+ "referer": "https://x.com/matthewisabel/status/2014444098690613718",+ "album_id": None,+ "datetime": 1769115874,+ "download_folder": "re:@matthewisabel (Twitter)/2026-01-22 - 2014444098690613718",+ }+ ],+ )+]