best way to save what you love
Svelte collaborationdownloaderinstagramjavascriptmusicredditsocial-mediasoundcloudsveltetiktoktwittertypescriptvideovimeovkwebappyoutubeyoutube-downloader
api/twitter: use newer graphql endpoint, refactor (#1436)
jjbec77b99e52e
diff --git a/api/src/processing/services/twitter.js b/api/src/processing/services/twitter.jsindex 8bbe0eeb..572bc2f0 100644--- a/api/src/processing/services/twitter.js+++ b/api/src/processing/services/twitter.js@@ -3,12 +3,12 @@ import { genericUserAgent } from "../../config.js";import { createStream } from "../../stream/manage.js";import { getCookie, updateCookie } from "../cookie/manager.js";-const graphqlURL = 'https://api.x.com/graphql/I9GDzyCGZL2wSoYFFrrTVw/TweetResultByRestId';+const graphqlURL = 'https://api.x.com/graphql/4Siu98E55GquhG52zHdY5w/TweetDetail';const tokenURL = 'https://api.x.com/1.1/guest/activate.json';-const tweetFeatures = JSON.stringify({"creator_subscriptions_tweet_preview_api_enabled":true,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"articles_preview_enabled":true,"tweetypie_unmention_optimization_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"rweb_video_timestamps_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"rweb_tipjar_consumption_enabled":true,"responsive_web_graphql_exclude_directive_enabled":true,"verified_phone_label_enabled":false,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_enhance_cards_enabled":false});+const tweetFeatures = JSON.stringify({"rweb_video_screen_enabled":false,"payments_enabled":false,"rweb_xchat_enabled":false,"profile_label_improvements_pcf_label_in_post_enabled":true,"rweb_tipjar_consumption_enabled":true,"verified_phone_label_enabled":false,"creator_subscriptions_tweet_preview_api_enabled":true,"responsive_web_graphql_timeline_navigation_enabled":true,"responsive_web_graphql_skip_user_profile_image_extensions_enabled":false,"premium_content_api_read_enabled":false,"communities_web_enable_tweet_community_results_fetch":true,"c9s_tweet_anatomy_moderator_badge_enabled":true,"responsive_web_grok_analyze_button_fetch_trends_enabled":false,"responsive_web_grok_analyze_post_followups_enabled":true,"responsive_web_jetfuel_frame":true,"responsive_web_grok_share_attachment_enabled":true,"articles_preview_enabled":true,"responsive_web_edit_tweet_api_enabled":true,"graphql_is_translatable_rweb_tweet_is_translatable_enabled":true,"view_counts_everywhere_api_enabled":true,"longform_notetweets_consumption_enabled":true,"responsive_web_twitter_article_tweet_consumption_enabled":true,"tweet_awards_web_tipping_enabled":false,"responsive_web_grok_show_grok_translated_post":false,"responsive_web_grok_analysis_button_from_backend":true,"creator_subscriptions_quote_tweet_preview_enabled":false,"freedom_of_speech_not_reach_fetch_enabled":true,"standardized_nudges_misinfo":true,"tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled":true,"longform_notetweets_rich_text_read_enabled":true,"longform_notetweets_inline_media_enabled":true,"responsive_web_grok_image_annotation_enabled":true,"responsive_web_grok_imagine_annotation_enabled":true,"responsive_web_grok_community_note_auto_translation_is_enabled":false,"responsive_web_enhance_cards_enabled":false});-const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false});+const tweetFieldToggles = JSON.stringify({"withArticleRichContentState":true,"withArticlePlainText":false,"withGrokAnalyze":false,"withDisallowedReplyControls":false});const commonHeaders = {"user-agent": genericUserAgent,@@ -100,10 +100,14 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {graphqlTweetURL.searchParams.set('variables',JSON.stringify({- tweetId,- withCommunity: false,- includePromotedContent: false,- withVoice: false+ focalTweetId: tweetId,+ with_rux_injections: false,+ rankingMode: "Relevance",+ includePromotedContent: true,+ withCommunity: true,+ withQuickPromoteEligibilityTweetFields: true,+ withBirdwatchNotes: true,+ withVoice: true}));graphqlTweetURL.searchParams.set('features', tweetFeatures);@@ -129,24 +133,48 @@ const requestTweet = async(dispatcher, tweetId, token, cookie) => {return result}-const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) => {- let tweetTypename = tweet?.data?.tweetResult?.result?.__typename;+const parseCard = (cardOuter) => {+ const card = JSON.parse(+ (cardOuter?.legacy?.binding_values[0].value+ || cardOuter?.binding_values?.unified_card)?.string_value,+ );++ if (!["video_website", "image_website"].includes(card?.type)+ || !card?.media_entities+ || card?.component_objects?.media_1?.type !== "media") {+ return;+ }++ const mediaId = card.component_objects?.media_1?.data?.id;+ return [card.media_entities[mediaId]];+};++const extractGraphqlMedia = async (thread, dispatcher, id, guestToken, cookie) => {+ const addInsn = thread?.data?.threaded_conversation_with_injections_v2?.instructions?.find(+ insn => insn.type === 'TimelineAddEntries'+ );++ const tweetResult = addInsn?.entries?.find(+ entry => entry.entryId === `tweet-${id}`+ )?.content?.itemContent?.tweet_results?.result;++ let tweetTypename = tweetResult?.__typename;if (!tweetTypename) {return { error: "fetch.empty" }}- if (tweetTypename === "TweetUnavailable") {- const reason = tweet?.data?.tweetResult?.result?.reason;- switch(reason) {- case "Protected":- return { error: "content.post.private" };- case "NsfwLoggedOut":- if (cookie) {- tweet = await requestTweet(dispatcher, id, guestToken, cookie);- tweet = await tweet.json();- tweetTypename = tweet?.data?.tweetResult?.result?.__typename;- } else return { error: "content.post.age" };+ if (tweetTypename === "TweetUnavailable" || tweetTypename === "TweetTombstone") {+ const reason = tweetResult?.result?.reason;+ if (reason === 'Protected') {+ return { error: "content.post.private" };+ } else if (reason === "NsfwLoggedOut" || tweetResult?.tombstone?.text?.text?.startsWith('Age-restricted')) {+ if (!cookie) {+ return { error: "content.post.age" };+ }++ const tweet = await requestTweet(dispatcher, id, guestToken, cookie).then(t => t.json());+ return extractGraphqlMedia(tweet, dispatcher, id, guestToken);}}@@ -154,8 +182,7 @@ const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) =>return { error: "content.post.unavailable" }}- let tweetResult = tweet.data.tweetResult.result,- baseTweet = tweetResult.legacy,+ let baseTweet = tweetResult.legacy,repostedTweet = baseTweet?.retweeted_status_result?.result.legacy.extended_entities;if (tweetTypename === "TweetWithVisibilityResults") {@@ -164,81 +191,51 @@ const extractGraphqlMedia = async (tweet, dispatcher, id, guestToken, cookie) =>}if (tweetResult.card?.legacy?.binding_values?.length) {- const card = JSON.parse(tweetResult.card.legacy.binding_values[0].value?.string_value);-- if (!["video_website", "image_website"].includes(card?.type) ||- !card?.media_entities ||- card?.component_objects?.media_1?.type !== "media") {- return;- }-- const mediaId = card.component_objects?.media_1?.data?.id;- return [card.media_entities[mediaId]];+ return parseCard(tweetResult.card);}return (repostedTweet?.media || baseTweet?.extended_entities?.media);}-const testResponse = (result) => {- const contentLength = result.headers.get("content-length");-- if (!contentLength || contentLength === '0') {- return false;- }-- if (!result.headers.get("content-type").startsWith("application/json")) {- return false;- }-- return true;-}-export default async function({ id, index, toGif, dispatcher, alwaysProxy, subtitleLang }) {const cookie = await getCookie('twitter');- let syndication = false;-let guestToken = await getGuestToken(dispatcher);if (!guestToken) return { error: "fetch.fail" };- // for now we assume that graphql api will come back after some time,- // so we try it first-let tweet = await requestTweet(dispatcher, id, guestToken);- // get new token & retry if old one expired- if ([403, 429].includes(tweet.status)) {- guestToken = await getGuestToken(dispatcher, true);- if (cookie) {- tweet = await requestTweet(dispatcher, id, guestToken, cookie);- } else {- tweet = await requestTweet(dispatcher, id, guestToken);+ if ([403, 404, 429].includes(tweet.status)) {+ // get new token & retry if old one expired+ if ([403, 429].includes(tweet.status)) {+ guestToken = await getGuestToken(dispatcher, true);}+ tweet = await requestTweet(dispatcher, id, guestToken, cookie);}- const testGraphql = testResponse(tweet);+ let media;+ try {+ tweet = await tweet.json();+ media = await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);+ } catch {}// if graphql requests fail, then resort to tweet embed api- if (!testGraphql) {- syndication = true;- tweet = await requestSyndication(dispatcher, id);+ if (!media || 'error' in media) {+ try {+ tweet = await requestSyndication(dispatcher, id);+ tweet = await tweet.json();- const testSyndication = testResponse(tweet);+ if (tweet?.card) {+ media = parseCard(tweet.card);+ }+ } catch {}- // if even syndication request failed, then cry out loud- if (!testSyndication) {- return { error: "fetch.fail" };- }+ media = tweet?.mediaDetails ?? media;}- tweet = await tweet.json();-- let media =- syndication- ? tweet.mediaDetails- : await extractGraphqlMedia(tweet, dispatcher, id, guestToken, cookie);-- if (!media) return { error: "fetch.empty" };+ if (!media || 'error' in media) {+ return { error: media?.error || "fetch.empty" };+ }// check if there's a video at given index (/video/<index>)if (index >= 0 && index < media?.length) {