Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

refactor(blocks): share one tweet-fetching helper across the three Twitter timeline blocks (#15780)

Reinier van der Leerbc837e4d6048
diff --git a/autogpt_platform/backend/backend/blocks/twitter/tweets/timeline.py b/autogpt_platform/backend/backend/blocks/twitter/tweets/timeline.py
index e20921cb36..7e622b3c85 100644
--- a/autogpt_platform/backend/backend/blocks/twitter/tweets/timeline.py
+++ b/autogpt_platform/backend/backend/blocks/twitter/tweets/timeline.py
@@ -1,5 +1,5 @@
-from datetime import datetime
-from typing import cast
+from collections.abc import Callable, Iterator
+from typing import Any, cast
import tweepy
from tweepy.client import Response
@@ -27,16 +27,7 @@ from backend.blocks.twitter._serializer import (
IncludesSerializer,
ResponseDataSerializer,
)
-from backend.blocks.twitter._types import (
- ExpansionFilter,
- TweetExpansionInputs,
- TweetFieldsFilter,
- TweetMediaFieldsFilter,
- TweetPlaceFieldsFilter,
- TweetPollFieldsFilter,
- TweetTimeWindowInputs,
- TweetUserFieldsFilter,
-)
+from backend.blocks.twitter._types import TweetExpansionInputs, TweetTimeWindowInputs
from backend.blocks.twitter.tweepy_exceptions import handle_tweepy_exception
from backend.data.model import SchemaField
@@ -148,97 +139,13 @@ class TwitterGetUserMentionsBlock(Block):
)
@staticmethod
- def get_mentions(
- credentials: TwitterCredentials,
- user_id: str,
- max_results: int | None,
- start_time: datetime | None,
- end_time: datetime | None,
- since_id: str | None,
- until_id: str | None,
- sort_order: str | None,
- pagination: str | None,
- expansions: ExpansionFilter | None,
- media_fields: TweetMediaFieldsFilter | None,
- place_fields: TweetPlaceFieldsFilter | None,
- poll_fields: TweetPollFieldsFilter | None,
- tweet_fields: TweetFieldsFilter | None,
- user_fields: TweetUserFieldsFilter | None,
- ):
- try:
- client = tweepy.Client(
- bearer_token=credentials.access_token.get_secret_value()
- )
-
- params = {
- "id": user_id,
- "max_results": max_results,
- "pagination_token": None if pagination == "" else pagination,
- "user_auth": False,
- }
-
- # Adding expansions to params If required by the user
- params = (
- TweetExpansionsBuilder(params)
- .add_expansions(expansions)
- .add_media_fields(media_fields)
- .add_place_fields(place_fields)
- .add_poll_fields(poll_fields)
- .add_tweet_fields(tweet_fields)
- .add_user_fields(user_fields)
- .build()
- )
-
- # Adding time window to params If required by the user
- params = (
- TweetDurationBuilder(params)
- .add_start_time(start_time)
- .add_end_time(end_time)
- .add_since_id(since_id)
- .add_until_id(until_id)
- .add_sort_order(sort_order)
- .build()
- )
-
- response = cast(
- Response,
- client.get_users_mentions(**params),
- )
-
- if not response.data and not response.meta:
- raise Exception("No tweets found")
-
- included = IncludesSerializer.serialize(response.includes)
- data = ResponseDataSerializer.serialize_list(response.data)
- meta = response.meta or {}
- next_token = meta.get("next_token", "")
-
- tweet_ids = []
- tweet_texts = []
- user_ids = []
- user_names = []
-
- if response.data:
- tweet_ids = [str(tweet.id) for tweet in response.data]
- tweet_texts = [tweet.text for tweet in response.data]
-
- if "users" in included:
- user_ids = [str(user["id"]) for user in included["users"]]
- user_names = [user["username"] for user in included["users"]]
-
- return (
- tweet_ids,
- tweet_texts,
- user_ids,
- user_names,
- data,
- included,
- meta,
- next_token,
- )
-
- except tweepy.TweepyException:
- raise
+ def get_mentions(credentials: TwitterCredentials, input_data: Input):
+ return _get_tweets(
+ credentials,
+ tweepy.Client.get_users_mentions,
+ input_data,
+ id=input_data.user_id,
+ )
async def run(
self,
@@ -248,42 +155,8 @@ class TwitterGetUserMentionsBlock(Block):
**kwargs,
) -> BlockOutput:
try:
- ids, texts, user_ids, user_names, data, included, meta, next_token = (
- self.get_mentions(
- credentials,
- input_data.user_id,
- input_data.max_results,
- input_data.start_time,
- input_data.end_time,
- input_data.since_id,
- input_data.until_id,
- input_data.sort_order,
- input_data.pagination_token,
- input_data.expansions,
- input_data.media_fields,
- input_data.place_fields,
- input_data.poll_fields,
- input_data.tweet_fields,
- input_data.user_fields,
- )
- )
- if ids:
- yield "ids", ids
- if texts:
- yield "texts", texts
- if user_ids:
- yield "userIds", user_ids
- if user_names:
- yield "userNames", user_names
- if next_token:
- yield "next_token", next_token
- if data:
- yield "data", data
- if included:
- yield "included", included
- if meta:
- yield "meta", meta
-
+ for output in _tweet_outputs(self.get_mentions(credentials, input_data)):
+ yield output
except Exception as e:
yield "error", handle_tweepy_exception(e)
@@ -383,95 +256,12 @@ class TwitterGetHomeTimelineBlock(Block):
)
@staticmethod
- def get_timeline(
- credentials: TwitterCredentials,
- max_results: int | None,
- start_time: datetime | None,
- end_time: datetime | None,
- since_id: str | None,
- until_id: str | None,
- sort_order: str | None,
- pagination: str | None,
- expansions: ExpansionFilter | None,
- media_fields: TweetMediaFieldsFilter | None,
- place_fields: TweetPlaceFieldsFilter | None,
- poll_fields: TweetPollFieldsFilter | None,
- tweet_fields: TweetFieldsFilter | None,
- user_fields: TweetUserFieldsFilter | None,
- ):
- try:
- client = tweepy.Client(
- bearer_token=credentials.access_token.get_secret_value()
- )
-
- params = {
- "max_results": max_results,
- "pagination_token": None if pagination == "" else pagination,
- "user_auth": False,
- }
-
- # Adding expansions to params If required by the user
- params = (
- TweetExpansionsBuilder(params)
- .add_expansions(expansions)
- .add_media_fields(media_fields)
- .add_place_fields(place_fields)
- .add_poll_fields(poll_fields)
- .add_tweet_fields(tweet_fields)
- .add_user_fields(user_fields)
- .build()
- )
-
- # Adding time window to params If required by the user
- params = (
- TweetDurationBuilder(params)
- .add_start_time(start_time)
- .add_end_time(end_time)
- .add_since_id(since_id)
- .add_until_id(until_id)
- .add_sort_order(sort_order)
- .build()
- )
-
- response = cast(
- Response,
- client.get_home_timeline(**params),
- )
-
- if not response.data and not response.meta:
- raise Exception("No tweets found")
-
- included = IncludesSerializer.serialize(response.includes)
- data = ResponseDataSerializer.serialize_list(response.data)
- meta = response.meta or {}
- next_token = meta.get("next_token", "")
-
- tweet_ids = []
- tweet_texts = []
- user_ids = []
- user_names = []
-
- if response.data:
- tweet_ids = [str(tweet.id) for tweet in response.data]
- tweet_texts = [tweet.text for tweet in response.data]
-
- if "users" in included:
- user_ids = [str(user["id"]) for user in included["users"]]
- user_names = [user["username"] for user in included["users"]]
-
- return (
- tweet_ids,
- tweet_texts,
- user_ids,
- user_names,
- data,
- included,
- meta,
- next_token,
- )
-
- except tweepy.TweepyException:
- raise
+ def get_timeline(credentials: TwitterCredentials, input_data: Input):
+ return _get_tweets(
+ credentials,
+ tweepy.Client.get_home_timeline,
+ input_data,
+ )
async def run(
self,
@@ -481,41 +271,8 @@ class TwitterGetHomeTimelineBlock(Block):
**kwargs,
) -> BlockOutput:
try:
- ids, texts, user_ids, user_names, data, included, meta, next_token = (
- self.get_timeline(
- credentials,
- input_data.max_results,
- input_data.start_time,
- input_data.end_time,
- input_data.since_id,
- input_data.until_id,
- input_data.sort_order,
- input_data.pagination_token,
- input_data.expansions,
- input_data.media_fields,
- input_data.place_fields,
- input_data.poll_fields,
- input_data.tweet_fields,
- input_data.user_fields,
- )
- )
- if ids:
- yield "ids", ids
- if texts:
- yield "texts", texts
- if user_ids:
- yield "userIds", user_ids
- if user_names:
- yield "userNames", user_names
- if next_token:
- yield "next_token", next_token
- if data:
- yield "data", data
- if included:
- yield "included", included
- if meta:
- yield "meta", meta
-
+ for output in _tweet_outputs(self.get_timeline(credentials, input_data)):
+ yield output
except Exception as e:
yield "error", handle_tweepy_exception(e)
@@ -627,97 +384,13 @@ class TwitterGetUserTweetsBlock(Block):
)
@staticmethod
- def get_user_tweets(
- credentials: TwitterCredentials,
- user_id: str,
- max_results: int | None,
- start_time: datetime | None,
- end_time: datetime | None,
- since_id: str | None,
- until_id: str | None,
- sort_order: str | None,
- pagination: str | None,
- expansions: ExpansionFilter | None,
- media_fields: TweetMediaFieldsFilter | None,
- place_fields: TweetPlaceFieldsFilter | None,
- poll_fields: TweetPollFieldsFilter | None,
- tweet_fields: TweetFieldsFilter | None,
- user_fields: TweetUserFieldsFilter | None,
- ):
- try:
- client = tweepy.Client(
- bearer_token=credentials.access_token.get_secret_value()
- )
-
- params = {
- "id": user_id,
- "max_results": max_results,
- "pagination_token": None if pagination == "" else pagination,
- "user_auth": False,
- }
-
- # Adding expansions to params If required by the user
- params = (
- TweetExpansionsBuilder(params)
- .add_expansions(expansions)
- .add_media_fields(media_fields)
- .add_place_fields(place_fields)
- .add_poll_fields(poll_fields)
- .add_tweet_fields(tweet_fields)
- .add_user_fields(user_fields)
- .build()
- )
-
- # Adding time window to params If required by the user
- params = (
- TweetDurationBuilder(params)
- .add_start_time(start_time)
- .add_end_time(end_time)
- .add_since_id(since_id)
- .add_until_id(until_id)
- .add_sort_order(sort_order)
- .build()
- )
-
- response = cast(
- Response,
- client.get_users_tweets(**params),
- )
-
- if not response.data and not response.meta:
- raise Exception("No tweets found")
-
- included = IncludesSerializer.serialize(response.includes)
- data = ResponseDataSerializer.serialize_list(response.data)
- meta = response.meta or {}
- next_token = meta.get("next_token", "")
-
- tweet_ids = []
- tweet_texts = []
- user_ids = []
- user_names = []
-
- if response.data:
- tweet_ids = [str(tweet.id) for tweet in response.data]
- tweet_texts = [tweet.text for tweet in response.data]
-
- if "users" in included:
- user_ids = [str(user["id"]) for user in included["users"]]
- user_names = [user["username"] for user in included["users"]]
-
- return (
- tweet_ids,
- tweet_texts,
- user_ids,
- user_names,
- data,
- included,
- meta,
- next_token,
- )
-
- except tweepy.TweepyException:
- raise
+ def get_user_tweets(credentials: TwitterCredentials, input_data: Input):
+ return _get_tweets(
+ credentials,
+ tweepy.Client.get_users_tweets,
+ input_data,
+ id=input_data.user_id,
+ )
async def run(
self,
@@ -727,41 +400,98 @@ class TwitterGetUserTweetsBlock(Block):
**kwargs,
) -> BlockOutput:
try:
- ids, texts, user_ids, user_names, data, included, meta, next_token = (
- self.get_user_tweets(
- credentials,
- input_data.user_id,
- input_data.max_results,
- input_data.start_time,
- input_data.end_time,
- input_data.since_id,
- input_data.until_id,
- input_data.sort_order,
- input_data.pagination_token,
- input_data.expansions,
- input_data.media_fields,
- input_data.place_fields,
- input_data.poll_fields,
- input_data.tweet_fields,
- input_data.user_fields,
- )
- )
- if ids:
- yield "ids", ids
- if texts:
- yield "texts", texts
- if user_ids:
- yield "userIds", user_ids
- if user_names:
- yield "userNames", user_names
- if next_token:
- yield "next_token", next_token
- if data:
- yield "data", data
- if included:
- yield "included", included
- if meta:
- yield "meta", meta
-
+ for output in _tweet_outputs(self.get_user_tweets(credentials, input_data)):
+ yield output
except Exception as e:
yield "error", handle_tweepy_exception(e)
+
+
+def _get_tweets(
+ credentials: TwitterCredentials,
+ fetch: Callable[..., Any],
+ input_data: "TwitterGetUserMentionsBlock.Input | TwitterGetHomeTimelineBlock.Input | TwitterGetUserTweetsBlock.Input",
+ **id_param: str,
+):
+ client = tweepy.Client(bearer_token=credentials.access_token.get_secret_value())
+
+ params = {
+ **id_param,
+ "max_results": input_data.max_results,
+ "pagination_token": (
+ None if input_data.pagination_token == "" else input_data.pagination_token
+ ),
+ "user_auth": False,
+ }
+ params = (
+ TweetExpansionsBuilder(params)
+ .add_expansions(input_data.expansions)
+ .add_media_fields(input_data.media_fields)
+ .add_place_fields(input_data.place_fields)
+ .add_poll_fields(input_data.poll_fields)
+ .add_tweet_fields(input_data.tweet_fields)
+ .add_user_fields(input_data.user_fields)
+ .build()
+ )
+ params = (
+ TweetDurationBuilder(params)
+ .add_start_time(input_data.start_time)
+ .add_end_time(input_data.end_time)
+ .add_since_id(input_data.since_id)
+ .add_until_id(input_data.until_id)
+ .add_sort_order(input_data.sort_order)
+ .build()
+ )
+
+ response = cast(Response, fetch(client, **params))
+
+ if not response.data and not response.meta:
+ raise Exception("No tweets found")
+
+ included = IncludesSerializer.serialize(response.includes)
+ data = ResponseDataSerializer.serialize_list(response.data)
+ meta = response.meta or {}
+ next_token = meta.get("next_token", "")
+
+ tweet_ids = []
+ tweet_texts = []
+ user_ids = []
+ user_names = []
+
+ if response.data:
+ tweet_ids = [str(tweet.id) for tweet in response.data]
+ tweet_texts = [tweet.text for tweet in response.data]
+
+ if "users" in included:
+ user_ids = [str(user["id"]) for user in included["users"]]
+ user_names = [user["username"] for user in included["users"]]
+
+ return (
+ tweet_ids,
+ tweet_texts,
+ user_ids,
+ user_names,
+ data,
+ included,
+ meta,
+ next_token,
+ )
+
+
+def _tweet_outputs(result: tuple) -> Iterator[tuple[str, Any]]:
+ ids, texts, user_ids, user_names, data, included, meta, next_token = result
+ if ids:
+ yield "ids", ids
+ if texts:
+ yield "texts", texts
+ if user_ids:
+ yield "userIds", user_ids
+ if user_names:
+ yield "userNames", user_names
+ if next_token:
+ yield "next_token", next_token
+ if data:
+ yield "data", data
+ if included:
+ yield "included", included
+ if meta:
+ yield "meta", meta
diff --git a/autogpt_platform/backend/backend/blocks/twitter/tweets/timeline_test.py b/autogpt_platform/backend/backend/blocks/twitter/tweets/timeline_test.py
new file mode 100644
index 0000000000..66319202b7
--- /dev/null
+++ b/autogpt_platform/backend/backend/blocks/twitter/tweets/timeline_test.py
@@ -0,0 +1,124 @@
+import pytest
+import tweepy
+from tweepy.client import Response
+
+from backend.blocks.twitter._auth import TEST_CREDENTIALS, TEST_CREDENTIALS_INPUT
+from backend.blocks.twitter.tweets.timeline import (
+ TwitterGetHomeTimelineBlock,
+ TwitterGetUserMentionsBlock,
+ TwitterGetUserTweetsBlock,
+)
+
+# test_block.py skips these blocks (they need a live account), so this is their only run.
+BLOCKS = [
+ (TwitterGetUserMentionsBlock, "get_users_mentions", {"id": "12345"}),
+ (TwitterGetHomeTimelineBlock, "get_home_timeline", {}),
+ (TwitterGetUserTweetsBlock, "get_users_tweets", {"id": "12345"}),
+]
+
+FULL_RESPONSE = Response(
+ data=[
+ tweepy.Tweet({"id": "11", "text": "one", "edit_history_tweet_ids": ["11"]}),
+ tweepy.Tweet({"id": "12", "text": "two", "edit_history_tweet_ids": ["12"]}),
+ ],
+ includes={
+ "users": [
+ tweepy.User({"id": "21", "name": "A", "username": "a"}),
+ tweepy.User({"id": "22", "name": "B", "username": "b"}),
+ ]
+ },
+ errors=[],
+ meta={"next_token": "nt", "result_count": 2},
+)
+
+
+def fake_client(monkeypatch):
+ calls: list[tuple[str, dict]] = []
+ answer = {"response": FULL_RESPONSE}
+
+ class FakeClient:
+ def __init__(self, **kwargs):
+ pass
+
+ def _answer(self, endpoint: str, params: dict):
+ calls.append((endpoint, params))
+ return answer["response"]
+
+ def get_users_mentions(self, **params):
+ return self._answer("get_users_mentions", params)
+
+ def get_home_timeline(self, **params):
+ return self._answer("get_home_timeline", params)
+
+ def get_users_tweets(self, **params):
+ return self._answer("get_users_tweets", params)
+
+ monkeypatch.setattr(tweepy, "Client", FakeClient)
+ return calls, answer
+
+
+async def _run(block_cls, **inputs):
+ raw = {"credentials": TEST_CREDENTIALS_INPUT, **inputs}
+ if "user_id" in block_cls.Input.model_fields:
+ raw["user_id"] = "12345"
+ return [
+ output
+ async for output in block_cls().run(
+ block_cls.Input(**raw), credentials=TEST_CREDENTIALS
+ )
+ ]
+
+
[email protected]("block_cls, endpoint, id_param", BLOCKS)
+async def test_each_block_calls_its_own_endpoint(
+ fake_client, block_cls, endpoint, id_param
+):
+ calls, _ = fake_client
+
+ await _run(block_cls, pagination_token="tok")
+
+ assert calls == [
+ (
+ endpoint,
+ {
+ **id_param,
+ "max_results": 10,
+ "pagination_token": "tok",
+ "user_auth": False,
+ },
+ )
+ ]
+
+
[email protected]("block_cls, endpoint, id_param", BLOCKS)
+async def test_outputs_follow_the_response(fake_client, block_cls, endpoint, id_param):
+ outputs = await _run(block_cls)
+
+ assert [name for name, _ in outputs] == [
+ "ids",
+ "texts",
+ "userIds",
+ "userNames",
+ "next_token",
+ "data",
+ "included",
+ "meta",
+ ]
+ values = dict(outputs)
+ assert values["ids"] == ["11", "12"]
+ assert values["texts"] == ["one", "two"]
+ assert values["userIds"] == ["21", "22"]
+ assert values["userNames"] == ["a", "b"]
+ assert values["next_token"] == "nt"
+ assert values["meta"] == {"next_token": "nt", "result_count": 2}
+
+
[email protected]("block_cls, endpoint, id_param", BLOCKS)
+async def test_an_empty_response_yields_an_error(
+ fake_client, block_cls, endpoint, id_param
+):
+ _, answer = fake_client
+ answer["response"] = Response(data=None, includes={}, errors=[], meta={})
+
+ assert await _run(block_cls) == [("error", "Unexpected error: No tweets found")]