Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add twitter post support (#1633)

NTFSvolume03d1dc01cc18
diff --git a/cyberdrop_dl/constants.py b/cyberdrop_dl/constants.py
index 0c2805149..04b75a74f 100644
--- a/cyberdrop_dl/constants.py
+++ b/cyberdrop_dl/constants.py
@@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, Any, Final
from aiohttp.resolver import AsyncResolver, ThreadedResolver
from rich.text import Text
+from cyberdrop_dl import env
from cyberdrop_dl.compat import Enum, StrEnum
if TYPE_CHECKING:
@@ -63,7 +64,6 @@ class TempExt(StrEnum):
class BlockedDomains:
partial_match = (
"facebook",
- "twitter.com",
"instagram",
"fbcdn",
"gfycat",
@@ -77,9 +77,13 @@ class BlockedDomains:
"beacons.page",
"beacons.ai",
"allmylinks.com",
- ".x.com",
)
- exact_match = ("x.com",)
+
+ exact_match = ()
+
+ if not env.ENABLE_TWITTER:
+ partial_match = *partial_match, "twitter.com", ".x.com"
+ exact_match = *exact_match, "x.com"
DEFAULT_APP_STORAGE = Path("./AppData")
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index 6fcd75689..8cdd5ee7a 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -127,6 +127,7 @@ from .transflix import TransflixCrawler
from .tubepornclassic import TubePornClassicCrawler
from .turbovid import TurboVidCrawler
from .twitch import TwitchCrawler
+from .twitter import TwitterCrawler
from .twitter_images import TwimgCrawler
from .twpornstars import TwPornstarsCrawler
from .txxx import TXXXCrawler
@@ -156,7 +157,7 @@ ALL_CRAWLERS: set[type[Crawler]] = {
crawler for name, crawler in globals().items() if name.endswith("Crawler") and crawler is not Crawler
}
ALL_CRAWLERS.update(WP_CRAWLERS, GENERIC_CRAWLERS, FORUM_CRAWLERS)
-DEBUG_CRAWLERS = {SimpCityCrawler}
+DEBUG_CRAWLERS = {SimpCityCrawler, TwitterCrawler}
if env.ENABLE_DEBUG_CRAWLERS == "d396ab8c85fcb1fecd22c8d9b58acf944a44e6d35014e9dd39e42c9a64091eda":
CRAWLERS = ALL_CRAWLERS
else:
diff --git a/cyberdrop_dl/crawlers/twitter.py b/cyberdrop_dl/crawlers/twitter.py
new file mode 100644
index 000000000..99c551166
--- /dev/null
+++ b/cyberdrop_dl/crawlers/twitter.py
@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils.utilities import error_handling_wrapper
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+
+
+_API_URL = AbsoluteHttpURL("https://api.fxtwitter.com")
+
+
+class TwitterCrawler(Crawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Tweet": "/<handle>/status/<tweet_id>",
+ }
+ OLD_DOMAINS: ClassVar[tuple[str, ...]] = ("twitter.com",)
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://x.com")
+ DOMAIN: ClassVar[str] = "twitter"
+ DEFAULT_POST_TITLE_FORMAT: ClassVar[str] = "{date:%Y-%m-%d} - {id}"
+
+ @property
+ def separate_posts(self) -> bool:
+ return True
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case [_user, "status", _]:
+ return await self.tweet(scrape_item)
+ case _:
+ raise ValueError
+
+ @error_handling_wrapper
+ async def tweet(self, scrape_item: ScrapeItem) -> None:
+ url = _API_URL / scrape_item.url.path.removeprefix("/")
+ tweet: dict[str, Any] = (await self.request_json(url))["tweet"]
+ scrape_item.possible_datetime = tweet["created_timestamp"]
+ name = tweet["author"]["screen_name"]
+ post_title = self.create_separate_post_title(None, tweet["id"], scrape_item.possible_datetime)
+ scrape_item.setup_as_profile(self.create_title(f"@{name}"))
+ scrape_item.add_to_parent_title(post_title)
+
+ await self.write_metadata(scrape_item, tweet["id"], tweet)
+
+ for media in tweet["media"]["all"]:
+ if media["type"] == "video":
+ media = max(media["formats"], key=lambda f: f.get("bitrate", 0))
+
+ source = self.parse_url(media["url"])
+ new_item = scrape_item.create_child(source)
+ self.handle_external_links(new_item, reset=False)
+ scrape_item.add_children()
diff --git a/cyberdrop_dl/crawlers/twitter_images.py b/cyberdrop_dl/crawlers/twitter_images.py
index cda752d6e..87ef71596 100644
--- a/cyberdrop_dl/crawlers/twitter_images.py
+++ b/cyberdrop_dl/crawlers/twitter_images.py
@@ -43,6 +43,8 @@ class TwimgCrawler(Crawler):
async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None = None) -> None:
_, *media_item.fallbacks = list(_make_download_urls(media_item.url))
+ if media_item.referer == media_item.url and media_item.parents:
+ media_item.referer = media_item.parents[0]
await super().handle_media_item(media_item, m3u8)
diff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.py
index bf8bb52ec..d2949d434 100644
--- a/cyberdrop_dl/env.py
+++ b/cyberdrop_dl/env.py
@@ -22,3 +22,4 @@ NO_PLUGINS = bool(os.getenv("CDL_NO_PLUGINS"))
PIXELDRAIN_PROXY = os.getenv("CDL_PIXELDRAIN_PROXY")
BANDCAMP_FORMATS = os.getenv("CDL_BANDCAMP_FORMATS")
+ENABLE_TWITTER = bool(os.getenv("CDL_ENABLE_TWITTER"))
diff --git a/tests/crawlers/test_cases/twitter.py b/tests/crawlers/test_cases/twitter.py
new file mode 100644
index 000000000..b6759dce9
--- /dev/null
+++ b/tests/crawlers/test_cases/twitter.py
@@ -0,0 +1,18 @@
+DOMAIN = "twitter"
+TEST_CASES = [
+ (
+ "https://x.com/matthewisabel/status/2014444098690613718",
+ [
+ {
+ "url": "https://video.twimg.com/amplify_video/2014443278259609602/vid/avc1/2744x2160/KfifaGUZoO2lsD79.mp4?tag=21",
+ "filename": "KfifaGUZoO2lsD79.mp4",
+ "debrid_link": None,
+ "original_filename": "KfifaGUZoO2lsD79.mp4",
+ "referer": "https://x.com/matthewisabel/status/2014444098690613718",
+ "album_id": None,
+ "datetime": 1769115874,
+ "download_folder": "re:@matthewisabel (Twitter)/2026-01-22 - 2014444098690613718",
+ }
+ ],
+ )
+]