Skip to content

Self-improving AI Employees for everyone

refactor(plugin): Wave 11.D.8a — twitter/crawlee/apify inlined

trohitg59bdb7e3bce2
diff --git a/server/nodes/scraper/apify_actor.py b/server/nodes/scraper/apify_actor.py
index 1c6f20bc..f29eef07 100644
--- a/server/nodes/scraper/apify_actor.py
+++ b/server/nodes/scraper/apify_actor.py
@@ -1,28 +1,133 @@
-"""Apify Actor — Wave 11.C migration."""
+"""Apify Actor — Wave 11.D.8 inlined.
+
+Runs Apify actors (Instagram, TikTok, Twitter, LinkedIn, Google, Crawler)
+via the official apify-client SDK. Quick-input helpers merge into the
+raw ``actorInput`` JSON for common actors.
+"""
from __future__ import annotations
-from typing import Any, Dict, Optional
+import json
+from typing import Any, Dict, List, Optional
from pydantic import BaseModel, ConfigDict, Field
+from core.logging import get_logger
from services.plugin import ActionNode, NodeContext, Operation, TaskQueue
+logger = get_logger(__name__)
+
+
+async def _get_apify_client():
+ from apify_client import ApifyClientAsync # lazy — optional dep
+ """Return an authenticated Apify client, or None if no token saved."""
+ from core.container import container
+ auth_service = container.auth_service()
+ api_token = await auth_service.get_api_key("apify", "default")
+ if not api_token:
+ return None
+ return ApifyClientAsync(api_token)
+
+
+def _build_actor_input(parameters: Dict[str, Any]) -> Dict[str, Any]:
+ """Merge actor quick-input helpers into the raw ``actorInput`` JSON."""
+ actor_id = parameters.get('actorId', '')
+ if actor_id == 'custom':
+ actor_id = parameters.get('customActorId', '')
+
+ actor_input = parameters.get('actorInput', '{}')
+ if isinstance(actor_input, str):
+ try:
+ actor_input = json.loads(actor_input) if actor_input.strip() else {}
+ except json.JSONDecodeError:
+ actor_input = {}
+ elif not isinstance(actor_input, dict):
+ actor_input = {}
+
+ if actor_id == 'apify/instagram-scraper':
+ urls = parameters.get('instagramUrls', '')
+ if urls:
+ actor_input['directUrls'] = [u.strip() for u in urls.split(',') if u.strip()]
+ elif actor_id == 'clockworks/tiktok-scraper':
+ profiles = parameters.get('tiktokProfiles', '')
+ hashtags = parameters.get('tiktokHashtags', '')
+ if profiles:
+ actor_input['profiles'] = [p.strip() for p in profiles.split(',') if p.strip()]
+ if hashtags:
+ actor_input['hashtags'] = [h.strip() for h in hashtags.split(',') if h.strip()]
+ elif actor_id == 'apidojo/tweet-scraper':
+ search_terms = parameters.get('twitterSearchTerms', '')
+ handles = parameters.get('twitterHandles', '')
+ if search_terms:
+ actor_input['searchTerms'] = [t.strip() for t in search_terms.split(',') if t.strip()]
+ if handles:
+ actor_input['twitterHandles'] = [h.strip() for h in handles.split(',') if h.strip()]
+ elif actor_id == 'apify/google-search-scraper':
+ query = parameters.get('googleSearchQuery', '')
+ pages = parameters.get('googleSearchPages', 1)
+ if query:
+ actor_input['searchQuery'] = query
+ actor_input['maxPagesPerQuery'] = pages
+ elif actor_id == 'apify/website-content-crawler':
+ start_urls = parameters.get('crawlerStartUrls', '')
+ max_depth = parameters.get('crawlerMaxDepth', 2)
+ max_pages = parameters.get('crawlerMaxPages', 50)
+ if start_urls:
+ actor_input['startUrls'] = [{'url': u.strip()} for u in start_urls.split(',') if u.strip()]
+ actor_input['maxCrawlDepth'] = max_depth
+ actor_input['maxCrawlPages'] = max_pages
+
+ return actor_input
+
+
+async def validate_apify_token(api_token: str) -> Dict[str, Any]:
+ """Validate an Apify API token by fetching /users/me.
+
+ Used by the websocket ``validate_apify_key`` handler and by the
+ Credentials modal. Lives on the plugin so there's no handler-file
+ ghost left behind after Wave 11.D.8.
+ """
+ try:
+ from apify_client import ApifyClientAsync # lazy — optional dep
+ client = ApifyClientAsync(api_token)
+ user_info = await client.user("me").get()
+ if not user_info:
+ return {"valid": False, "error": "Could not fetch user info"}
+ plan = user_info.get("plan")
+ return {
+ "valid": True,
+ "username": user_info.get("username", ""),
+ "email": user_info.get("email", ""),
+ "plan": plan.get("id", "free") if isinstance(plan, dict) else "free",
+ }
+ except Exception as e:
+ logger.error(f"[Apify] Token validation error: {e}")
+ msg = str(e)
+ if "401" in msg or "Unauthorized" in msg:
+ return {"valid": False, "error": "Invalid API token"}
+ return {"valid": False, "error": msg}
+
class ApifyActorParams(BaseModel):
- actor_id: str = Field(..., alias="actorId")
- actor_input: Dict[str, Any] = Field(default_factory=dict, alias="actorInput")
- max_results: int = Field(default=20, alias="maxResults", ge=1, le=10000)
+ actor_id: str = Field(default="", alias="actorId")
+ actor_input: Any = Field(default_factory=dict, alias="actorInput")
+ max_results: int = Field(default=100, alias="maxResults", ge=1, le=10000)
timeout: int = Field(default=300, ge=1, le=3600)
- memory: int = Field(default=512, ge=128, le=8192)
+ memory: int = Field(default=1024, ge=128, le=8192)
model_config = ConfigDict(populate_by_name=True, extra="allow")
class ApifyActorOutput(BaseModel):
- items: Optional[list] = None
- count: Optional[int] = None
run_id: Optional[str] = None
+ actor_id: Optional[str] = None
+ status: Optional[str] = None
+ items: Optional[list] = None
+ item_count: Optional[int] = None
+ dataset_id: Optional[str] = None
+ compute_units: Optional[float] = None
+ started_at: Optional[str] = None
+ finished_at: Optional[str] = None
model_config = ConfigDict(extra="allow")
@@ -48,12 +153,63 @@ class ApifyActorNode(ActionNode):
Output = ApifyActorOutput
@Operation("run")
- async def run(self, ctx: NodeContext, params: ApifyActorParams) -> Any:
- from services.handlers.apify import handle_apify_actor
- response = await handle_apify_actor(
- node_id=ctx.node_id, node_type=self.type,
- parameters=params.model_dump(by_alias=True), context=ctx.raw,
+ async def run(self, ctx: NodeContext, params: ApifyActorParams) -> ApifyActorOutput:
+ client = await _get_apify_client()
+ if not client:
+ raise RuntimeError(
+ "Apify API token not configured. Please add your token in Credentials.",
+ )
+
+ p = params.model_dump(by_alias=True)
+ actor_id = p.get("actorId", "")
+ if actor_id == "custom":
+ actor_id = p.get("customActorId", "")
+ if not actor_id:
+ raise RuntimeError("Actor ID is required")
+
+ actor_input = _build_actor_input(p)
+ timeout_secs = int(p.get("timeout", 300))
+ max_results = int(p.get("maxResults", 100))
+ memory_mbytes = int(p.get("memory", 1024))
+
+ logger.info(
+ f"[Apify] Running actor {actor_id} "
+ f"timeout={timeout_secs}s memory={memory_mbytes}MB",
+ )
+ run_info = await client.actor(actor_id).call(
+ run_input=actor_input,
+ timeout_secs=timeout_secs,
+ memory_mbytes=memory_mbytes,
+ )
+
+ if run_info is None:
+ raise RuntimeError("Actor run failed - no result returned")
+
+ status = run_info.get("status", "UNKNOWN")
+ run_id = run_info.get("id", "")
+ dataset_id = run_info.get("defaultDatasetId", "")
+
+ if status == "FAILED":
+ raise RuntimeError(run_info.get("errorMessage", "Actor run failed"))
+ if status == "TIMED-OUT":
+ raise RuntimeError("Actor timed out. Try increasing the timeout.")
+ if status == "ABORTED":
+ raise RuntimeError("Actor run was aborted")
+
+ items: List[Dict[str, Any]] = []
+ if dataset_id:
+ listing = await client.dataset(dataset_id).list_items(limit=max_results)
+ items = listing.items if listing else []
+ logger.info(f"[Apify] Actor {actor_id} completed: {len(items)} items")
+
+ return ApifyActorOutput(
+ run_id=run_id,
+ actor_id=actor_id,
+ status=status,
+ items=items,
+ item_count=len(items),
+ dataset_id=dataset_id,
+ compute_units=run_info.get("usageTotalUsd", 0),
+ started_at=run_info.get("startedAt", ""),
+ finished_at=run_info.get("finishedAt", ""),
)
- if response.get("success"):
- return response.get("result") or response
- raise RuntimeError(response.get("error") or "Apify run failed")
diff --git a/server/nodes/scraper/crawlee_scraper.py b/server/nodes/scraper/crawlee_scraper.py
index 25f757c4..e0bc574c 100644
--- a/server/nodes/scraper/crawlee_scraper.py
+++ b/server/nodes/scraper/crawlee_scraper.py
@@ -1,16 +1,175 @@
-"""Crawlee Scraper — Wave 11.C migration."""
+"""Crawlee Scraper — Wave 11.D.8 inlined.
+
+Thin wrapper over ``crawlee.BeautifulSoupCrawler`` (static HTML) and
+``crawlee.PlaywrightCrawler`` (JS-rendered). Crawlee handles concurrency,
+retries, storage, and anti-bot internally.
+"""
from __future__ import annotations
-from typing import Any, Literal, Optional
+import asyncio
+from datetime import timedelta
+from typing import Any, Dict, List, Literal, Optional
+from urllib.parse import urljoin
from pydantic import BaseModel, ConfigDict, Field
+from core.logging import get_logger
from services.plugin import ActionNode, NodeContext, Operation, TaskQueue
+logger = get_logger(__name__)
+
+_MAX_CONTENT_LENGTH = 100_000
+
+
+async def _get_proxy_config(parameters: Dict[str, Any], url: str):
+ """Bridge MachinaOs ProxyService to Crawlee ProxyConfiguration."""
+ if not parameters.get('useProxy', False):
+ return None
+ try:
+ from services.proxy import get_proxy_service
+ proxy_svc = get_proxy_service()
+ proxy_url = await proxy_svc.get_proxy_url(url, parameters)
+ if not proxy_url:
+ return None
+ from crawlee.proxy_configuration import ProxyConfiguration
+ return ProxyConfiguration(proxy_urls=[proxy_url])
+ except Exception as e:
+ logger.warning(f"[Crawlee] Proxy setup failed, proceeding without: {e}")
+ return None
+
+
+def _extract_text(soup, css_selector: str, output_format: str) -> str:
+ target = soup.select(css_selector) if css_selector else [soup]
+ if output_format == 'html':
+ parts = [str(el) for el in target]
+ else:
+ parts = [el.get_text(separator='\n', strip=True) for el in target]
+ text = '\n\n'.join(parts)
+ if output_format == 'markdown':
+ try:
+ import html2text
+ h = html2text.HTML2Text()
+ h.ignore_links = False
+ h.ignore_images = False
+ h.body_width = 0
+ raw_html = '\n\n'.join(str(el) for el in target)
+ text = h.handle(raw_html)
+ except ImportError:
+ pass
+ return text[:_MAX_CONTENT_LENGTH]
+
+
+def _extract_links(soup, base_url: str) -> List[str]:
+ return [urljoin(base_url, a['href']) for a in soup.find_all('a', href=True)]
+
+
+async def _run_beautifulsoup(url, pages, p, proxy_config, css_selector, extract_links,
+ output_format, mode, max_pages, max_depth,
+ max_concurrency, timeout_secs):
+ from crawlee import ConcurrencySettings
+ from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
+ from crawlee.storage_clients import MemoryStorageClient
+
+ crawler = BeautifulSoupCrawler(
+ max_requests_per_crawl=max_pages,
+ max_crawl_depth=max_depth if mode == 'crawl' else 0,
+ request_handler_timeout=timedelta(seconds=timeout_secs),
+ concurrency_settings=ConcurrencySettings(
+ max_concurrency=max_concurrency,
+ desired_concurrency=min(max_concurrency, 10),
+ ),
+ storage_client=MemoryStorageClient(),
+ proxy_configuration=proxy_config,
+ configure_logging=False,
+ )
+ link_selector = p.get('linkSelector', '') or 'a[href]'
+ url_pattern = p.get('urlPattern', '')
+
+ @crawler.router.default_handler
+ async def handler(ctx: BeautifulSoupCrawlingContext) -> None:
+ title = ctx.soup.title.string if ctx.soup.title else ''
+ content = _extract_text(ctx.soup, css_selector, output_format)
+ page_data: Dict[str, Any] = {
+ 'url': ctx.request.url, 'title': title, 'content': content,
+ }
+ if extract_links:
+ page_data['links'] = _extract_links(ctx.soup, ctx.request.url)
+ pages.append(page_data)
+
+ if mode == 'crawl':
+ kwargs: Dict[str, Any] = {'selector': link_selector}
+ if url_pattern:
+ import re
+ from fnmatch import translate
+ kwargs['include'] = [re.compile(translate(url_pattern))]
+ await ctx.enqueue_links(**kwargs)
+
+ await asyncio.wait_for(crawler.run([url]), timeout=timeout_secs)
+
+
+async def _run_playwright(url, pages, p, proxy_config, css_selector, extract_links,
+ output_format, mode, max_pages, max_depth,
+ max_concurrency, timeout_secs):
+ from crawlee import ConcurrencySettings
+ from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
+ from crawlee.storage_clients import MemoryStorageClient
+
+ browser_type = p.get('browserType', 'chromium')
+ wait_for_selector = p.get('waitForSelector', '')
+ wait_timeout = p.get('waitTimeout', 30000)
+ take_screenshot = p.get('screenshot', False) or p.get('takeScreenshot', False)
+
+ crawler = PlaywrightCrawler(
+ browser_type=browser_type, headless=True,
+ max_requests_per_crawl=max_pages,
+ max_crawl_depth=max_depth if mode == 'crawl' else 0,
+ request_handler_timeout=timedelta(seconds=timeout_secs),
+ concurrency_settings=ConcurrencySettings(
+ max_concurrency=max_concurrency,
+ desired_concurrency=min(max_concurrency, 10),
+ ),
+ storage_client=MemoryStorageClient(),
+ proxy_configuration=proxy_config,
+ configure_logging=False,
+ )
+ link_selector = p.get('linkSelector', '') or 'a[href]'
+ url_pattern = p.get('urlPattern', '')
+
+ @crawler.router.default_handler
+ async def handler(ctx: PlaywrightCrawlingContext) -> None:
+ page = ctx.page
+ if wait_for_selector:
+ await page.wait_for_selector(wait_for_selector, timeout=wait_timeout)
+ title = await page.title()
+ html = await page.content()
+ from bs4 import BeautifulSoup
+ soup = BeautifulSoup(html, 'html.parser')
+ content = _extract_text(soup, css_selector, output_format)
+ page_data: Dict[str, Any] = {
+ 'url': ctx.request.url, 'title': title, 'content': content,
+ }
+ if extract_links:
+ page_data['links'] = _extract_links(soup, ctx.request.url)
+ if take_screenshot:
+ import base64
+ screenshot_bytes = await page.screenshot(type='png')
+ page_data['screenshot'] = base64.b64encode(screenshot_bytes).decode()
+ pages.append(page_data)
+
+ if mode == 'crawl':
+ kwargs: Dict[str, Any] = {'selector': link_selector}
+ if url_pattern:
+ import re
+ from fnmatch import translate
+ kwargs['include'] = [re.compile(translate(url_pattern))]
+ await ctx.enqueue_links(**kwargs)
+
+ await asyncio.wait_for(crawler.run([url]), timeout=timeout_secs)
+
class CrawleeScraperParams(BaseModel):
- url: str = Field(...)
+ url: str = Field(default="")
crawler_type: Literal["beautifulsoup", "playwright", "adaptive"] = Field(
default="beautifulsoup", alias="crawlerType",
)
@@ -19,13 +178,19 @@ class CrawleeScraperParams(BaseModel):
extract_links: bool = Field(default=False, alias="extractLinks")
max_pages: int = Field(default=10, alias="maxPages", ge=1, le=1000)
take_screenshot: bool = Field(default=False, alias="takeScreenshot")
+ output_format: Literal["text", "html", "markdown"] = Field(
+ default="text", alias="outputFormat",
+ )
model_config = ConfigDict(populate_by_name=True, extra="allow")
class CrawleeScraperOutput(BaseModel):
- items: Optional[list] = None
- pages_crawled: Optional[int] = None
+ pages: Optional[list] = None
+ page_count: Optional[int] = None
+ crawler_type: Optional[str] = None
+ mode: Optional[str] = None
+ proxied: Optional[bool] = None
model_config = ConfigDict(extra="allow")
@@ -51,12 +216,55 @@ class CrawleeScraperNode(ActionNode):
Output = CrawleeScraperOutput
@Operation("scrape")
- async def scrape(self, ctx: NodeContext, params: CrawleeScraperParams) -> Any:
- from services.handlers.crawlee import handle_crawlee_scraper
- response = await handle_crawlee_scraper(
- node_id=ctx.node_id, node_type=self.type,
- parameters=params.model_dump(by_alias=True), context=ctx.raw,
+ async def scrape(self, ctx: NodeContext, params: CrawleeScraperParams) -> CrawleeScraperOutput:
+ p = params.model_dump(by_alias=True)
+ url = p.get('url', '').strip()
+ if not url:
+ raise RuntimeError("URL is required")
+
+ crawler_type = p.get('crawlerType', 'beautifulsoup')
+ mode = p.get('mode', 'single')
+ css_selector = p.get('cssSelector', '')
+ extract_links = p.get('extractLinks', False)
+ output_format = p.get('outputFormat', 'text')
+ max_pages = p.get('maxPages', 10) if mode == 'crawl' else 1
+ max_depth = p.get('maxDepth', 2) if mode == 'crawl' else 0
+ max_concurrency = p.get('maxConcurrency', 5)
+ timeout_secs = p.get('timeout', 60)
+
+ pages: List[Dict[str, Any]] = []
+ proxy_config = await _get_proxy_config(p, url)
+
+ try:
+ if crawler_type in ('beautifulsoup', 'adaptive'):
+ await _run_beautifulsoup(
+ url, pages, p, proxy_config, css_selector, extract_links,
+ output_format, mode, max_pages, max_depth,
+ max_concurrency, timeout_secs,
+ )
+ elif crawler_type == 'playwright':
+ await _run_playwright(
+ url, pages, p, proxy_config, css_selector, extract_links,
+ output_format, mode, max_pages, max_depth,
+ max_concurrency, timeout_secs,
+ )
+ else:
+ raise RuntimeError(f"Unknown crawler type: {crawler_type}")
+ except ImportError as e:
+ msg = str(e).lower()
+ if 'playwright' in msg:
+ raise RuntimeError(
+ "Playwright not installed. Run: "
+ "pip install 'crawlee[playwright]' && playwright install chromium",
+ )
+ if 'crawlee' in msg:
+ raise RuntimeError(
+ "Crawlee not installed. Run: pip install 'crawlee[beautifulsoup]'",
+ )
+ raise
+
+ logger.info(f"[Crawlee] Scraped {len(pages)} page(s) from {url}")
+ return CrawleeScraperOutput(
+ pages=pages, page_count=len(pages),
+ crawler_type=crawler_type, mode=mode, proxied=proxy_config is not None,
)
- if response.get("success"):
- return response.get("result") or response
- raise RuntimeError(response.get("error") or "Crawlee scrape failed")
diff --git a/server/nodes/twitter/_base.py b/server/nodes/twitter/_base.py
new file mode 100644
index 00000000..10ec08fb
--- /dev/null
+++ b/server/nodes/twitter/_base.py
@@ -0,0 +1,311 @@
+"""Twitter/X shared helpers (Wave 11.D.8).
+
+Used by the four twitter plugins (send / search / user / receive).
+Wraps the XDK SDK with:
+- Lazy client auth (refresh on 401/403).
+- asyncio.to_thread for all sync XDK calls.
+- Pricing-service usage tracking.
+- Response / tweet / user formatters.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Any, Dict, List, Optional
+
+from core.logging import get_logger
+from services.pricing import get_pricing_service
+
+logger = get_logger(__name__)
+
+
+def _is_auth_error(e: Exception) -> bool:
+ s = str(e)
+ return any(code in s for code in ("401", "403", "Unauthorized", "Forbidden"))
+
+
+async def track_twitter_usage(
+ node_id: str, action: str, resource_count: int, context: Dict[str, Any],
+) -> Dict[str, float]:
+ """Record a Twitter API call in ``api_usage_metrics``."""
+ from core.container import container
+
+ pricing = get_pricing_service()
+ cost_data = pricing.calculate_api_cost('twitter', action, resource_count)
+
+ db = container.database()
+ await db.save_api_usage_metric({
+ 'session_id': context.get('session_id', 'default'),
+ 'node_id': node_id,
+ 'workflow_id': context.get('workflow_id'),
+ 'service': 'twitter',
+ 'operation': cost_data.get('operation', action),
+ 'endpoint': action,
+ 'resource_count': resource_count,
+ 'cost': cost_data.get('total_cost', 0.0),
+ })
+ return cost_data
+
+
+async def get_twitter_client():
+ """Build an XDK Client from the stored OAuth2 access token."""
+ from core.container import container
+ from xdk import Client
+
+ auth_service = container.auth_service()
+ tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")
+ if not tokens or not tokens.get("access_token"):
+ raise RuntimeError("Twitter not connected. Please authenticate via Credentials.")
+ return Client(access_token=tokens["access_token"])
+
+
+async def refresh_and_get_client():
+ """Refresh the OAuth2 token and return a new client."""
+ from core.container import container
+ from services.twitter_oauth import TwitterOAuth
+ from xdk import Client
+
+ auth_service = container.auth_service()
+ tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")
+ refresh_token = tokens.get("refresh_token", "") if tokens else ""
+ if not refresh_token:
+ raise RuntimeError("Twitter token expired. Please re-authenticate.")
+
+ client_id = await auth_service.get_api_key("twitter_client_id") or ""
+ client_secret = await auth_service.get_api_key("twitter_client_secret")
+
+ oauth = TwitterOAuth(client_id=client_id, client_secret=client_secret, redirect_uri="")
+ result = await oauth.refresh_access_token(refresh_token)
+ if not result.get("success"):
+ raise RuntimeError("Twitter token refresh failed. Please re-authenticate.")
+
+ new_access = result["access_token"]
+ new_refresh = result.get("refresh_token", refresh_token)
+
+ await auth_service.store_oauth_tokens(
+ provider="twitter",
+ access_token=new_access,
+ refresh_token=new_refresh,
+ email=tokens.get("email", ""),
+ name=tokens.get("name", ""),
+ scopes=tokens.get("scopes", ""),
+ customer_id="owner",
+ )
+ logger.info("Twitter token refreshed successfully")
+ return Client(access_token=new_access)
+
+
+async def call_with_retry(fn, *args, **kwargs):
+ """Run ``fn`` with an existing client; on auth failure, refresh and retry once.
+
+ ``fn`` is an async callable taking the client as its first argument:
+ ``async def fn(client, ...) -> Any``.
+ """
+ client = await get_twitter_client()
+ try:
+ return await fn(client, *args, **kwargs)
+ except Exception as e:
+ if _is_auth_error(e):
+ logger.info(f"Twitter auth error, refreshing token: {e}")
+ client = await refresh_and_get_client()
+ return await fn(client, *args, **kwargs)
+ raise
+
+
+async def get_my_user_id(client) -> str:
+ resp = await asyncio.to_thread(client.users.get_me)
+ return resp.data["id"]
+
+
+def format_response(response) -> Dict[str, Any]:
+ if hasattr(response, 'data'):
+ data = response.data
+ if isinstance(data, dict):
+ return data
+ if hasattr(data, '__dict__'):
+ return {k: v for k, v in data.__dict__.items() if not k.startswith('_')}
+ return {"data": str(data)}
+ return {"response": str(response)}
+
+
+def format_user(user) -> Dict[str, Any]:
+ def g(attr, default=None):
+ if isinstance(user, dict):
+ return user.get(attr, default)
+ return getattr(user, attr, default)
+ return {
+ "id": g("id"),
+ "username": g("username"),
+ "name": g("name"),
+ "profile_image_url": g("profile_image_url"),
+ "verified": g("verified", False),
+ "description": g("description"),
+ "created_at": str(g("created_at", "")),
+ }
+
+
+def format_tweet(
+ tweet,
+ users_by_id: Optional[Dict] = None,
+ media_by_key: Optional[Dict] = None,
+ tweets_by_id: Optional[Dict] = None,
+) -> Dict[str, Any]:
+ """Format a tweet with enriched data from search includes."""
+ users_by_id = users_by_id or {}
+ media_by_key = media_by_key or {}
+ tweets_by_id = tweets_by_id or {}
+
+ def g(obj, key, default=None):
+ return obj.get(key, default) if isinstance(obj, dict) else getattr(obj, key, default)
+
+ tweet_id = g(tweet, 'id')
+ author_id = g(tweet, 'author_id')
+ text = g(tweet, 'text', '')
+
+ entities = g(tweet, 'entities') or {}
+ if isinstance(entities, dict):
+ urls_list = entities.get('urls', [])
+ elif hasattr(entities, 'urls'):
+ urls_list = entities.urls or []
+ else:
+ urls_list = []
+
+ expanded_urls = []
+ display_text = text
+ for u in urls_list:
+ if isinstance(u, dict):
+ short = u.get('url', ''); expanded = u.get('expanded_url', ''); display = u.get('display_url', '')
+ else:
+ short = getattr(u, 'url', ''); expanded = getattr(u, 'expanded_url', ''); display = getattr(u, 'display_url', '')
+ if short and expanded:
+ expanded_urls.append({"url": short, "expanded_url": expanded, "display_url": display})
+ display_text = display_text.replace(short, expanded)
+
+ note_tweet = g(tweet, 'note_tweet')
+ if note_tweet:
+ note_text = note_tweet.get('text', '') if isinstance(note_tweet, dict) else getattr(note_tweet, 'text', '')
+ if note_text:
+ text = note_text
+ display_text = note_text
+
+ author_info = users_by_id.get(str(author_id)) if author_id else None
+
+ attachments = g(tweet, 'attachments') or {}
+ if isinstance(attachments, dict):
+ media_keys = attachments.get('media_keys', [])
+ elif hasattr(attachments, 'media_keys'):
+ media_keys = attachments.media_keys or []
+ else:
+ media_keys = []
+ media_list = [media_by_key[k] for k in media_keys if k in media_by_key]
+
+ ref_raw = g(tweet, 'referenced_tweets') or []
+ referenced = []
+ for ref in ref_raw:
+ if isinstance(ref, dict):
+ ref_type = ref.get('type', ''); ref_id = ref.get('id', '')
+ else:
+ ref_type = getattr(ref, 'type', ''); ref_id = getattr(ref, 'id', '')
+ ref_data = tweets_by_id.get(str(ref_id))
+ referenced.append({
+ "type": ref_type,
+ "id": ref_id,
+ "text": ref_data.get("text", "") if ref_data else None,
+ "author_id": ref_data.get("author_id") if ref_data else None,
+ })
+
+ metrics = g(tweet, 'public_metrics') or {}
+ if not isinstance(metrics, dict) and hasattr(metrics, 'model_dump'):
+ metrics = metrics.model_dump()
+ elif not isinstance(metrics, dict):
+ metrics = {}
+
+ result = {
+ "id": tweet_id,
+ "text": text,
+ "display_text": display_text,
+ "author_id": author_id,
+ "created_at": str(g(tweet, 'created_at', '')),
+ "lang": g(tweet, 'lang'),
+ "source": g(tweet, 'source'),
+ "conversation_id": g(tweet, 'conversation_id'),
+ "in_reply_to_user_id": g(tweet, 'in_reply_to_user_id'),
+ "possibly_sensitive": g(tweet, 'possibly_sensitive', False),
+ "public_metrics": metrics,
+ }
+ if author_info:
+ result["author"] = author_info
+ if expanded_urls:
+ result["urls"] = expanded_urls
+ if media_list:
+ result["media"] = media_list
+ if referenced:
+ result["referenced_tweets"] = referenced
+ return result
+
+
+def _includes_to_dict(raw) -> Dict[str, Any]:
+ if not raw:
+ return {}
+ if hasattr(raw, 'model_dump'):
+ try:
+ return raw.model_dump()
+ except Exception:
+ return {}
+ if isinstance(raw, dict):
+ return raw
+ return {}
+
+
+def sync_search_recent(client, query: str, max_results: int) -> Dict[str, Any]:
+ """Run ``posts.search_recent`` synchronously; returns {tweets, includes}."""
+ for page in client.posts.search_recent(
+ query=query, max_results=max_results,
+ tweet_fields=[
+ "author_id", "created_at", "entities", "public_metrics",
+ "possibly_sensitive", "lang", "source",
+ "conversation_id", "in_reply_to_user_id", "referenced_tweets",
+ "note_tweet",
+ ],
+ expansions=[
+ "author_id", "attachments.media_keys",
+ "referenced_tweets.id", "referenced_tweets.id.author_id",
+ ],
+ media_fields=["url", "preview_image_url", "type", "alt_text", "variants"],
+ user_fields=["username", "name", "profile_image_url"],
+ ):
+ return {
+ "tweets": getattr(page, 'data', []) or [],
+ "includes": _includes_to_dict(getattr(page, 'includes', None)),
+ }
+ return {"tweets": [], "includes": {}}
+
+
+def includes_lookups(includes: Dict[str, Any]):
+ """Build {users_by_id, media_by_key, tweets_by_id} from a search-includes dict."""
+ def _as_dict(obj):
+ if isinstance(obj, dict):
+ return obj
+ if hasattr(obj, '__dict__'):
+ return {k: v for k, v in obj.__dict__.items() if not k.startswith('_')}
+ return {}
+
+ users_by_id = {}
+ for u in (includes.get("users") or []):
+ uid = u.get("id") if isinstance(u, dict) else getattr(u, "id", None)
+ if uid:
+ users_by_id[str(uid)] = _as_dict(u)
+
+ media_by_key = {}
+ for m in (includes.get("media") or []):
+ mk = m.get("media_key") if isinstance(m, dict) else getattr(m, "media_key", None)
+ if mk:
+ media_by_key[mk] = _as_dict(m)
+
+ tweets_by_id = {}
+ for t in (includes.get("tweets") or []):
+ tid = t.get("id") if isinstance(t, dict) else getattr(t, "id", None)
+ if tid:
+ tweets_by_id[str(tid)] = _as_dict(t)
+
+ return users_by_id, media_by_key, tweets_by_id
diff --git a/server/nodes/twitter/twitter_search.py b/server/nodes/twitter/twitter_search.py
index c92965d8..d25590e2 100644
--- a/server/nodes/twitter/twitter_search.py
+++ b/server/nodes/twitter/twitter_search.py
@@ -1,16 +1,22 @@
-"""Twitter Search — Wave 11.C migration."""
+"""Twitter Search — Wave 11.D.8 inlined."""
from __future__ import annotations
-from typing import Any, Optional
+import asyncio
+from typing import Optional
from pydantic import BaseModel, ConfigDict, Field
from services.plugin import ActionNode, NodeContext, Operation, TaskQueue
+from ._base import (
+ call_with_retry, format_tweet, includes_lookups,
+ sync_search_recent, track_twitter_usage,
+)
+
class TwitterSearchParams(BaseModel):
- query: str = Field(..., min_length=1)
+ query: str = Field(default="", min_length=0)
max_results: int = Field(default=10, alias="maxResults", ge=10, le=100)
model_config = ConfigDict(populate_by_name=True, extra="ignore")
@@ -19,10 +25,20 @@ class TwitterSearchParams(BaseModel):
class TwitterSearchOutput(BaseModel):
tweets: Optional[list] = None
count: Optional[int] = None
+ query: Optional[str] = None
model_config = ConfigDict(extra="allow")
+async def _do_search(client, query: str, max_results: int, node_id: str, ctx_raw: dict) -> TwitterSearchOutput:
+ search = await asyncio.to_thread(sync_search_recent, client, query, max_results)
+ users_by_id, media_by_key, tweets_by_id = includes_lookups(search["includes"])
+ tweets = [format_tweet(t, users_by_id, media_by_key, tweets_by_id) for t in search["tweets"]]
+ if tweets:
+ await track_twitter_usage(node_id, "search", len(tweets), ctx_raw)
+ return TwitterSearchOutput(tweets=tweets, count=len(tweets), query=query)
+
+
class TwitterSearchNode(ActionNode):
type = "twitterSearch"
display_name = "Twitter Search"
@@ -46,12 +62,8 @@ class TwitterSearchNode(ActionNode):
Output = TwitterSearchOutput
@Operation("search", cost={"service": "twitter", "action": "search", "count": 1})
- async def search(self, ctx: NodeContext, params: TwitterSearchParams) -> Any:
- from services.handlers.twitter import handle_twitter_search
- response = await handle_twitter_search(
- node_id=ctx.node_id, node_type=self.type,
- parameters=params.model_dump(by_alias=True), context=ctx.raw,
- )
- if response.get("success"):
- return response.get("result") or response
- raise RuntimeError(response.get("error") or "Twitter search failed")
+ async def search(self, ctx: NodeContext, params: TwitterSearchParams) -> TwitterSearchOutput:
+ if not params.query:
+ raise RuntimeError("Search query is required")
+ max_results = max(10, min(params.max_results, 100))
+ return await call_with_retry(_do_search, params.query, max_results, ctx.node_id, ctx.raw)
diff --git a/server/nodes/twitter/twitter_send.py b/server/nodes/twitter/twitter_send.py
index 1d44e1b9..1846ce5b 100644
--- a/server/nodes/twitter/twitter_send.py
+++ b/server/nodes/twitter/twitter_send.py
@@ -1,13 +1,18 @@
-"""Twitter Send — Wave 11.C migration. Dual-purpose ActionNode + AI tool."""
+"""Twitter Send — Wave 11.D.8 inlined (tweet/reply/retweet/like/unlike/delete)."""
from __future__ import annotations
-from typing import Any, Literal, Optional
+import asyncio
+from typing import Literal, Optional
from pydantic import BaseModel, ConfigDict, Field
from services.plugin import ActionNode, NodeContext, Operation, TaskQueue
+from ._base import (
+ call_with_retry, format_response, get_my_user_id, track_twitter_usage,
+)
+
class TwitterSendParams(BaseModel):
action: Literal["tweet", "reply", "retweet", "like", "unlike", "delete"] = "tweet"
@@ -19,12 +24,69 @@ class TwitterSendParams(BaseModel):
class TwitterSendOutput(BaseModel):
- tweet_id: Optional[str] = None
- success: Optional[bool] = None
+ action: Optional[str] = None
+ data: Optional[dict] = None
model_config = ConfigDict(extra="allow")
+async def _do_send(client, action: str, p: dict, node_id: str, ctx_raw: dict) -> TwitterSendOutput:
+ if action == "tweet":
+ text = p.get("text", "")
+ if not text:
+ raise RuntimeError("Tweet text is required")
+ result = await asyncio.to_thread(client.posts.create, body={"text": text[:280]})
+ await track_twitter_usage(node_id, "tweet", 1, ctx_raw)
+ return TwitterSendOutput(action="tweet_sent", data=format_response(result))
+
+ if action == "reply":
+ text = p.get("text", "")
+ reply_to = p.get("replyToId") or p.get("reply_to_id")
+ if not text or not reply_to:
+ raise RuntimeError("Text and reply_to_id are required")
+ result = await asyncio.to_thread(
+ client.posts.create,
+ body={"text": text[:280], "reply": {"in_reply_to_tweet_id": reply_to}},
+ )
+ await track_twitter_usage(node_id, "reply", 1, ctx_raw)
+ return TwitterSendOutput(action="reply_sent", data=format_response(result))
+
+ tweet_id = p.get("tweetId") or p.get("tweet_id")
+ if not tweet_id:
+ raise RuntimeError("tweet_id is required")
+
+ if action == "retweet":
+ user_id = await get_my_user_id(client)
+ result = await asyncio.to_thread(
+ client.users.repost_post, user_id, body={"tweet_id": tweet_id},
+ )
+ await track_twitter_usage(node_id, "retweet", 1, ctx_raw)
+ return TwitterSendOutput(action="retweeted", data=format_response(result))
+
+ if action == "like":
+ user_id = await get_my_user_id(client)
+ result = await asyncio.to_thread(
+ client.users.like_post, user_id, body={"tweet_id": tweet_id},
+ )
+ await track_twitter_usage(node_id, "like", 1, ctx_raw)
+ return TwitterSendOutput(action="liked", data=format_response(result))
+
+ if action == "unlike":
+ user_id = await get_my_user_id(client)
+ result = await asyncio.to_thread(
+ client.users.unlike_post, user_id, tweet_id=tweet_id,
+ )
+ await track_twitter_usage(node_id, "unlike", 1, ctx_raw)
+ return TwitterSendOutput(action="unliked", data=format_response(result))
+
+ if action == "delete":
+ result = await asyncio.to_thread(client.posts.delete, tweet_id)
+ await track_twitter_usage(node_id, "delete", 1, ctx_raw)
+ return TwitterSendOutput(action="deleted", data=format_response(result))
+
+ raise RuntimeError(f"Unknown action: {action}")
+
+
class TwitterSendNode(ActionNode):
type = "twitterSend"
display_name = "Twitter Send"
@@ -48,12 +110,7 @@ class TwitterSendNode(ActionNode):
Output = TwitterSendOutput
@Operation("send", cost={"service": "twitter", "action": "send", "count": 1})
- async def send(self, ctx: NodeContext, params: TwitterSendParams) -> Any:
- from services.handlers.twitter import handle_twitter_send
- response = await handle_twitter_send(
- node_id=ctx.node_id, node_type=self.type,
- parameters=params.model_dump(by_alias=True), context=ctx.raw,
- )
- if response.get("success"):
- return response.get("result") or response
- raise RuntimeError(response.get("error") or "Twitter send failed")
+ async def send(self, ctx: NodeContext, params: TwitterSendParams) -> TwitterSendOutput:
+ p = params.model_dump(by_alias=True)
+ action = p.get("action", "tweet")
+ return await call_with_retry(_do_send, action, p, ctx.node_id, ctx.raw)
diff --git a/server/nodes/twitter/twitter_user.py b/server/nodes/twitter/twitter_user.py
index d16fe4bf..bb0d1655 100644
--- a/server/nodes/twitter/twitter_user.py
+++ b/server/nodes/twitter/twitter_user.py
@@ -1,13 +1,16 @@
-"""Twitter User — Wave 11.C migration."""
+"""Twitter User — Wave 11.D.8 inlined (me / by_username / by_id / followers / following)."""
from __future__ import annotations
-from typing import Any, Literal, Optional
+import asyncio
+from typing import Literal, Optional
from pydantic import BaseModel, ConfigDict, Field
from services.plugin import ActionNode, NodeContext, Operation, TaskQueue
+from ._base import call_with_retry, format_user, get_my_user_id, track_twitter_usage
+
class TwitterUserParams(BaseModel):
operation: Literal["me", "by_username", "by_id", "followers", "following"] = "me"
@@ -19,12 +22,79 @@ class TwitterUserParams(BaseModel):
class TwitterUserOutput(BaseModel):
+ operation: Optional[str] = None
user: Optional[dict] = None
users: Optional[list] = None
+ count: Optional[int] = None
model_config = ConfigDict(extra="allow")
+def _sync_followers(client, user_id, max_results):
+ for page in client.users.get_followers(
+ user_id, max_results=max_results, user_fields=["created_at"],
+ ):
+ return list(getattr(page, 'data', []) or [])
+ return []
+
+
+def _sync_following(client, user_id, max_results):
+ for page in client.users.get_following(
+ user_id, max_results=max_results, user_fields=["created_at"],
+ ):
+ return list(getattr(page, 'data', []) or [])
+ return []
+
+
+async def _do_lookup(client, op: str, p: dict, node_id: str, ctx_raw: dict) -> TwitterUserOutput:
+ if op == "me":
+ result = await asyncio.to_thread(
+ client.users.get_me, user_fields=["created_at", "description"],
+ )
+ await track_twitter_usage(node_id, "me", 1, ctx_raw)
+ return TwitterUserOutput(operation="me", user=format_user(result.data))
+
+ if op == "by_username":
+ username = p.get("username")
+ if not username:
+ raise RuntimeError("Username is required")
+ result = await asyncio.to_thread(
+ client.users.get_by_usernames, usernames=[username],
+ user_fields=["description", "created_at"],
+ )
+ users = getattr(result, 'data', []) or []
+ if not users:
+ raise RuntimeError(f"User @{username} not found")
+ await track_twitter_usage(node_id, "by_username", 1, ctx_raw)
+ return TwitterUserOutput(operation="by_username", user=format_user(users[0]))
+
+ if op == "by_id":
+ user_id = p.get("userId") or p.get("user_id")
+ if not user_id:
+ raise RuntimeError("User ID is required")
+ result = await asyncio.to_thread(
+ client.users.get_by_ids, ids=[user_id],
+ user_fields=["description", "created_at"],
+ )
+ users = getattr(result, 'data', []) or []
+ if not users:
+ raise RuntimeError(f"User ID {user_id} not found")
+ await track_twitter_usage(node_id, "by_id", 1, ctx_raw)
+ return TwitterUserOutput(operation="by_id", user=format_user(users[0]))
+
+ if op in ("followers", "following"):
+ user_id = p.get("userId") or p.get("user_id") or await get_my_user_id(client)
+ max_results = max(1, min(int(p.get("maxResults", 100)), 1000))
+ fetch = _sync_followers if op == "followers" else _sync_following
+ raw = await asyncio.to_thread(fetch, client, user_id, max_results)
+ users = [format_user(u) for u in raw]
+ if users:
+ await track_twitter_usage(node_id, op, len(users), ctx_raw)
+ return TwitterUserOutput(operation=op, users=users, count=len(users))
+
+ raise RuntimeError(f"Unknown operation: {op}")
+
+
class TwitterUserNode(ActionNode):
type = "twitterUser"
display_name = "Twitter User"
@@ -48,12 +118,7 @@ class TwitterUserNode(ActionNode):
Output = TwitterUserOutput
@Operation("lookup", cost={"service": "twitter", "action": "user_lookup", "count": 1})
- async def lookup(self, ctx: NodeContext, params: TwitterUserParams) -> Any:
- from services.handlers.twitter import handle_twitter_user
- response = await handle_twitter_user(
- node_id=ctx.node_id, node_type=self.type,
- parameters=params.model_dump(by_alias=True), context=ctx.raw,
- )
- if response.get("success"):
- return response.get("result") or response
- raise RuntimeError(response.get("error") or "Twitter user lookup failed")
+ async def lookup(self, ctx: NodeContext, params: TwitterUserParams) -> TwitterUserOutput:
+ p = params.model_dump(by_alias=True)
+ op = p.get("operation", "me")
+ return await call_with_retry(_do_lookup, op, p, ctx.node_id, ctx.raw)
diff --git a/server/routers/websocket.py b/server/routers/websocket.py
index 57b93738..29d7b3f9 100644
--- a/server/routers/websocket.py
+++ b/server/routers/websocket.py
@@ -1810,7 +1810,7 @@ async def handle_validate_maps_key(data: Dict[str, Any], websocket: WebSocket) -
async def handle_validate_apify_key(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
"""Validate Apify API token and save to database if valid."""
- from services.handlers.apify import validate_apify_token
+ from nodes.scraper.apify_actor import validate_apify_token
broadcaster = get_status_broadcaster()
auth_service = container.auth_service()
diff --git a/server/services/handlers/__init__.py b/server/services/handlers/__init__.py
index cad2cdb3..d677e20b 100644
--- a/server/services/handlers/__init__.py
+++ b/server/services/handlers/__init__.py
@@ -72,12 +72,7 @@ from .whatsapp import (
handle_whatsapp_db,
)
-# Twitter handlers
-from .twitter import (
- handle_twitter_send,
- handle_twitter_search,
- handle_twitter_user,
-)
+# Twitter handlers inlined into nodes/twitter/*.py (Wave 11.D.8).
# Google Workspace handlers inlined into nodes/google/*.py (Wave 11.D.4).
@@ -99,8 +94,7 @@ from .search import (
# Email handlers (Himalaya CLI)
from .email import handle_email_send, handle_email_read, handle_email_receive
-# Crawlee web scraping handler
-from .crawlee import handle_crawlee_scraper
+# Crawlee handler inlined into nodes/scraper/crawlee_scraper.py (Wave 11.D.8).
# Browser automation handler (agent-browser CLI)
from .browser import handle_browser
@@ -165,9 +159,6 @@ __all__ = [
'handle_whatsapp_send',
'handle_whatsapp_db',
# Twitter
- 'handle_twitter_send',
- 'handle_twitter_search',
- 'handle_twitter_user',
# Social
'handle_social_receive',
'handle_social_send',
@@ -175,8 +166,6 @@ __all__ = [
'handle_brave_search',
'handle_serper_search',
'handle_perplexity_search',
- # Crawlee
- 'handle_crawlee_scraper',
# RLM
'handle_rlm_agent',
# Deep Agent
diff --git a/server/services/handlers/apify.py b/server/services/handlers/apify.py
deleted file mode 100644
index 700c755f..00000000
--- a/server/services/handlers/apify.py
+++ /dev/null
@@ -1,261 +0,0 @@
-"""Apify node handlers using official apify-client SDK."""
-
-import json
-import time
-from typing import Dict, Any, Optional, List
-
-from apify_client import ApifyClientAsync
-
-from core.logging import get_logger
-
-logger = get_logger(__name__)
-
-
-async def _get_apify_client(context: Dict[str, Any]) -> Optional[ApifyClientAsync]:
- """Get authenticated Apify client from stored credentials."""
- from core.container import container
- auth_service = container.auth_service()
- api_token = await auth_service.get_api_key("apify", "default")
- if not api_token:
- return None
- return ApifyClientAsync(api_token)
-
-
-def _build_actor_input(parameters: Dict[str, Any]) -> Dict[str, Any]:
- """Build actor input from parameters, merging quick helpers with raw JSON input."""
- actor_id = parameters.get('actorId', '')
- if actor_id == 'custom':
- actor_id = parameters.get('customActorId', '')
-
- # Start with raw JSON input if provided
- actor_input = parameters.get('actorInput', '{}')
- if isinstance(actor_input, str):
- try:
- actor_input = json.loads(actor_input) if actor_input.strip() else {}
- except json.JSONDecodeError:
- actor_input = {}
- elif not isinstance(actor_input, dict):
- actor_input = {}
-
- # Merge quick helpers based on actor type
- if actor_id == 'apify/instagram-scraper':
- urls = parameters.get('instagramUrls', '')
- if urls:
- actor_input['directUrls'] = [u.strip() for u in urls.split(',') if u.strip()]
-
- elif actor_id == 'clockworks/tiktok-scraper':
- profiles = parameters.get('tiktokProfiles', '')
- hashtags = parameters.get('tiktokHashtags', '')
- if profiles:
- actor_input['profiles'] = [p.strip() for p in profiles.split(',') if p.strip()]
- if hashtags:
- actor_input['hashtags'] = [h.strip() for h in hashtags.split(',') if h.strip()]
-
- elif actor_id == 'apidojo/tweet-scraper':
- search_terms = parameters.get('twitterSearchTerms', '')
- handles = parameters.get('twitterHandles', '')
- if search_terms:
- actor_input['searchTerms'] = [t.strip() for t in search_terms.split(',') if t.strip()]
- if handles:
- actor_input['twitterHandles'] = [h.strip() for h in handles.split(',') if h.strip()]
-
- elif actor_id == 'apify/google-search-scraper':
- query = parameters.get('googleSearchQuery', '')
- pages = parameters.get('googleSearchPages', 1)
- if query:
- actor_input['searchQuery'] = query
- actor_input['maxPagesPerQuery'] = pages
-
- elif actor_id == 'apify/website-content-crawler':
- start_urls = parameters.get('crawlerStartUrls', '')
- max_depth = parameters.get('crawlerMaxDepth', 2)
- max_pages = parameters.get('crawlerMaxPages', 50)
- if start_urls:
- actor_input['startUrls'] = [{'url': u.strip()} for u in start_urls.split(',') if u.strip()]
- actor_input['maxCrawlDepth'] = max_depth
- actor_input['maxCrawlPages'] = max_pages
-
- return actor_input
-
-
-async def handle_apify_actor(
- node_id: str,
- node_type: str,
- parameters: Dict[str, Any],
- context: Dict[str, Any]
-) -> Dict[str, Any]:
- """Run an Apify actor and return results using official SDK.
-
- Args:
- node_id: Node ID
- node_type: Node type ('apifyActor')
- parameters: Node parameters including actorId, actorInput, maxResults, timeout
- context: Execution context
-
- Returns:
- Dict with success status, results, and execution metadata
- """
- start_time = time.time()
-
- try:
- # Get authenticated client
- client = await _get_apify_client(context)
- if not client:
- return {
- "success": False,
- "error": "Apify API token not configured. Please add your token in Credentials.",
- "execution_time": time.time() - start_time
- }
-
- # Get actor ID
- actor_id = parameters.get("actorId", "")
- if actor_id == "custom":
- actor_id = parameters.get("customActorId", "")
- if not actor_id:
- return {
- "success": False,
- "error": "Actor ID is required",
- "execution_time": time.time() - start_time
- }
-
- # Build actor input
- actor_input = _build_actor_input(parameters)
-
- # Get execution options
- timeout_secs = parameters.get("timeout", 300)
- max_results = parameters.get("maxResults", 100)
- memory_mbytes = parameters.get("memory", 1024)
-
- logger.info(f"[Apify] Running actor {actor_id} with timeout={timeout_secs}s, memory={memory_mbytes}MB")
-
- # Get actor client and run
- actor_client = client.actor(actor_id)
-
- # call() runs the actor and waits for it to finish
- # SDK handles polling and retries automatically
- run_info = await actor_client.call(
- run_input=actor_input,
- timeout_secs=timeout_secs,
- memory_mbytes=memory_mbytes
- )
-
- if run_info is None:
- return {
- "success": False,
- "error": "Actor run failed - no result returned",
- "execution_time": time.time() - start_time
- }
-
- run_status = run_info.get("status", "UNKNOWN")
- run_id = run_info.get("id", "")
- dataset_id = run_info.get("defaultDatasetId", "")
-
- # Handle non-success statuses
- if run_status == "FAILED":
- error_msg = run_info.get("errorMessage", "Actor run failed")
- logger.error(f"[Apify] Actor {actor_id} failed: {error_msg}")
- return {
- "success": False,
- "error": error_msg,
- "result": {
- "run_id": run_id,
- "actor_id": actor_id,
- "status": run_status
- },
- "execution_time": time.time() - start_time
- }
- elif run_status == "TIMED-OUT":
- logger.warning(f"[Apify] Actor {actor_id} timed out")
- return {
- "success": False,
- "error": "Actor timed out. Try increasing the timeout.",
- "result": {
- "run_id": run_id,
- "actor_id": actor_id,
- "status": run_status
- },
- "execution_time": time.time() - start_time
- }
- elif run_status == "ABORTED":
- return {
- "success": False,
- "error": "Actor run was aborted",
- "result": {
- "run_id": run_id,
- "actor_id": actor_id,
- "status": run_status
- },
- "execution_time": time.time() - start_time
- }
-
- # Get results from dataset
- items: List[Dict[str, Any]] = []
- if dataset_id:
- dataset_client = client.dataset(dataset_id)
- list_result = await dataset_client.list_items(limit=max_results)
- items = list_result.items if list_result else []
- logger.info(f"[Apify] Actor {actor_id} completed: {len(items)} items retrieved")
-
- # Build result
- result = {
- "run_id": run_id,
- "actor_id": actor_id,
- "status": run_status,
- "items": items,
- "item_count": len(items),
- "dataset_id": dataset_id,
- "compute_units": run_info.get("usageTotalUsd", 0),
- "started_at": run_info.get("startedAt", ""),
- "finished_at": run_info.get("finishedAt", "")
- }
-
- return {
- "success": True,
- "result": result,
- "execution_time": time.time() - start_time
- }
-
- except Exception as e:
- logger.error(f"[Apify] Error: {e}")
- error_msg = str(e)
- # Improve error messages for common cases
- if "401" in error_msg or "Unauthorized" in error_msg:
- error_msg = "Invalid Apify API token. Please check your credentials."
- elif "404" in error_msg or "not found" in error_msg.lower():
- error_msg = "Actor not found. Check the actor ID."
- return {
- "success": False,
- "error": error_msg,
- "execution_time": time.time() - start_time
- }
-
-
-async def validate_apify_token(api_token: str) -> Dict[str, Any]:
- """Validate Apify API token by fetching user info.
-
- Args:
- api_token: Apify API token to validate
-
- Returns:
- Dict with 'valid' boolean and user info if valid
- """
- try:
- client = ApifyClientAsync(api_token)
- user_client = client.user("me")
- user_info = await user_client.get()
-
- if user_info:
- return {
- "valid": True,
- "username": user_info.get("username", ""),
- "email": user_info.get("email", ""),
- "plan": user_info.get("plan", {}).get("id", "free") if isinstance(user_info.get("plan"), dict) else "free"
- }
- return {"valid": False, "error": "Could not fetch user info"}
-
- except Exception as e:
- logger.error(f"[Apify] Token validation error: {e}")
- error_msg = str(e)
- if "401" in error_msg or "Unauthorized" in error_msg:
- return {"valid": False, "error": "Invalid API token"}
- return {"valid": False, "error": str(e)}
diff --git a/server/services/handlers/crawlee.py b/server/services/handlers/crawlee.py
deleted file mode 100644
index 6f359d4a..00000000
--- a/server/services/handlers/crawlee.py
+++ /dev/null
@@ -1,324 +0,0 @@
-"""Crawlee web scraping handler using the crawlee library.
-
-Thin wrapper around Crawlee's BeautifulSoupCrawler and PlaywrightCrawler.
-The library handles concurrency, retries, storage, and anti-bot internally.
-"""
-
-import asyncio
-import time
-from datetime import timedelta
-from typing import Any, Dict, List, Optional
-
-from core.logging import get_logger
-
-logger = get_logger(__name__)
-
-# Max content size per page to prevent memory issues
-_MAX_CONTENT_LENGTH = 100_000
-
-
-async def _get_proxy_config(parameters: Dict[str, Any], url: str):
- """Bridge MachinaOs ProxyService to Crawlee ProxyConfiguration.
-
- Returns a crawlee.ProxyConfiguration or None.
- """
- if not parameters.get('useProxy', False):
- return None
-
- try:
- from services.proxy import get_proxy_service
- proxy_svc = get_proxy_service()
- proxy_url = await proxy_svc.get_proxy_url(url, parameters)
- if not proxy_url:
- return None
-
- from crawlee.proxy_configuration import ProxyConfiguration
- return ProxyConfiguration(proxy_urls=[proxy_url])
- except Exception as e:
- logger.warning(f"[Crawlee] Proxy setup failed, proceeding without: {e}")
- return None
-
-
-def _extract_text(soup, css_selector: str, output_format: str) -> str:
- """Extract content from BeautifulSoup object."""
- target = soup.select(css_selector) if css_selector else [soup]
-
- if output_format == 'html':
- parts = [str(el) for el in target]
- else:
- parts = [el.get_text(separator='\n', strip=True) for el in target]
-
- text = '\n\n'.join(parts)
-
- if output_format == 'markdown':
- try:
- import html2text
- h = html2text.HTML2Text()
- h.ignore_links = False
- h.ignore_images = False
- h.body_width = 0
- raw_html = '\n\n'.join(str(el) for el in target)
- text = h.handle(raw_html)
- except ImportError:
- pass # Fall back to plain text
-
- return text[:_MAX_CONTENT_LENGTH]
-
-
-def _extract_links(soup, base_url: str) -> List[str]:
- """Extract all links from page."""
- from urllib.parse import urljoin
- links = []
- for a in soup.find_all('a', href=True):
- links.append(urljoin(base_url, a['href']))
- return links
-
-
-async def handle_crawlee_scraper(
- node_id: str,
- node_type: str,
- parameters: Dict[str, Any],
- context: Dict[str, Any]
-) -> Dict[str, Any]:
- """Scrape web pages using Crawlee (BeautifulSoup or Playwright).
-
- Args:
- node_id: Node ID
- node_type: Node type ('crawleeScraper')
- parameters: Node parameters
- context: Execution context
-
- Returns:
- Dict with success status, scraped pages, and metadata
- """
- start_time = time.time()
-
- url = parameters.get('url', '').strip()
- if not url:
- return {
- "success": False,
- "error": "URL is required",
- "execution_time": time.time() - start_time
- }
-
- crawler_type = parameters.get('crawlerType', 'beautifulsoup')
- mode = parameters.get('mode', 'single')
- css_selector = parameters.get('cssSelector', '')
- extract_links = parameters.get('extractLinks', False)
- output_format = parameters.get('outputFormat', 'text')
- max_pages = parameters.get('maxPages', 10) if mode == 'crawl' else 1
- max_depth = parameters.get('maxDepth', 2) if mode == 'crawl' else 0
- max_concurrency = parameters.get('maxConcurrency', 5)
- timeout_secs = parameters.get('timeout', 60)
-
- # Collect results from crawler handlers
- pages: List[Dict[str, Any]] = []
-
- try:
- proxy_config = await _get_proxy_config(parameters, url)
-
- if crawler_type in ('beautifulsoup', 'adaptive'):
- await _run_beautifulsoup(
- url, pages, parameters, proxy_config,
- css_selector, extract_links, output_format,
- mode, max_pages, max_depth, max_concurrency, timeout_secs,
- )
- elif crawler_type == 'playwright':
- await _run_playwright(
- url, pages, parameters, proxy_config,
- css_selector, extract_links, output_format,
- mode, max_pages, max_depth, max_concurrency, timeout_secs,
- )
- else:
- return {
- "success": False,
- "error": f"Unknown crawler type: {crawler_type}",
- "execution_time": time.time() - start_time
- }
-
- logger.info(f"[Crawlee] Scraped {len(pages)} page(s) from {url}")
-
- return {
- "success": True,
- "result": {
- "pages": pages,
- "page_count": len(pages),
- "crawler_type": crawler_type,
- "mode": mode,
- "proxied": proxy_config is not None,
- },
- "execution_time": time.time() - start_time
- }
-
- except ImportError as e:
- msg = str(e)
- if 'playwright' in msg.lower():
- msg = ("Playwright not installed. Run: pip install 'crawlee[playwright]' && playwright install chromium")
- elif 'crawlee' in msg.lower():
- msg = ("Crawlee not installed. Run: pip install 'crawlee[beautifulsoup]'")
- return {
- "success": False,
- "error": msg,
- "execution_time": time.time() - start_time
- }
- except Exception as e:
- logger.error(f"[Crawlee] Error scraping {url}: {e}")
- return {
- "success": False,
- "error": str(e),
- "execution_time": time.time() - start_time
- }
-
-
-async def _run_beautifulsoup(
- url: str,
- pages: List[Dict],
- parameters: Dict[str, Any],
- proxy_config,
- css_selector: str,
- extract_links: bool,
- output_format: str,
- mode: str,
- max_pages: int,
- max_depth: int,
- max_concurrency: int,
- timeout_secs: int,
-) -> None:
- """Run BeautifulSoupCrawler and collect results into pages list."""
- from crawlee import ConcurrencySettings
- from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
- from crawlee.storage_clients import MemoryStorageClient
-
- crawler = BeautifulSoupCrawler(
- max_requests_per_crawl=max_pages,
- max_crawl_depth=max_depth if mode == 'crawl' else 0,
- request_handler_timeout=timedelta(seconds=timeout_secs),
- concurrency_settings=ConcurrencySettings(
- max_concurrency=max_concurrency,
- desired_concurrency=min(max_concurrency, 10),
- ),
- storage_client=MemoryStorageClient(),
- proxy_configuration=proxy_config,
- configure_logging=False,
- )
-
- link_selector = parameters.get('linkSelector', '') or 'a[href]'
- url_pattern = parameters.get('urlPattern', '')
-
- @crawler.router.default_handler
- async def handler(ctx: BeautifulSoupCrawlingContext) -> None:
- title = ctx.soup.title.string if ctx.soup.title else ''
- content = _extract_text(ctx.soup, css_selector, output_format)
-
- page_data: Dict[str, Any] = {
- 'url': ctx.request.url,
- 'title': title,
- 'content': content,
- }
- if extract_links:
- page_data['links'] = _extract_links(ctx.soup, ctx.request.url)
-
- pages.append(page_data)
-
- # Follow links in crawl mode
- if mode == 'crawl':
- enqueue_kwargs: Dict[str, Any] = {'selector': link_selector}
- if url_pattern:
- import re
- from fnmatch import translate
- pattern = re.compile(translate(url_pattern))
- enqueue_kwargs['include'] = [pattern]
- await ctx.enqueue_links(**enqueue_kwargs)
-
- await asyncio.wait_for(
- crawler.run([url]),
- timeout=timeout_secs,
- )
-
-
-async def _run_playwright(
- url: str,
- pages: List[Dict],
- parameters: Dict[str, Any],
- proxy_config,
- css_selector: str,
- extract_links: bool,
- output_format: str,
- mode: str,
- max_pages: int,
- max_depth: int,
- max_concurrency: int,
- timeout_secs: int,
-) -> None:
- """Run PlaywrightCrawler and collect results into pages list."""
- from crawlee import ConcurrencySettings
- from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
- from crawlee.storage_clients import MemoryStorageClient
-
- browser_type = parameters.get('browserType', 'chromium')
- wait_for_selector = parameters.get('waitForSelector', '')
- wait_timeout = parameters.get('waitTimeout', 30000)
- take_screenshot = parameters.get('screenshot', False)
-
- crawler = PlaywrightCrawler(
- browser_type=browser_type,
- headless=True,
- max_requests_per_crawl=max_pages,
- max_crawl_depth=max_depth if mode == 'crawl' else 0,
- request_handler_timeout=timedelta(seconds=timeout_secs),
- concurrency_settings=ConcurrencySettings(
- max_concurrency=max_concurrency,
- desired_concurrency=min(max_concurrency, 10),
- ),
- storage_client=MemoryStorageClient(),
- proxy_configuration=proxy_config,
- configure_logging=False,
- )
-
- link_selector = parameters.get('linkSelector', '') or 'a[href]'
- url_pattern = parameters.get('urlPattern', '')
-
- @crawler.router.default_handler
- async def handler(ctx: PlaywrightCrawlingContext) -> None:
- page = ctx.page
-
- # Wait for specific element if configured
- if wait_for_selector:
- await page.wait_for_selector(wait_for_selector, timeout=wait_timeout)
-
- title = await page.title()
-
- # Get page content for extraction
- html = await page.content()
- from bs4 import BeautifulSoup
- soup = BeautifulSoup(html, 'html.parser')
- content = _extract_text(soup, css_selector, output_format)
-
- page_data: Dict[str, Any] = {
- 'url': ctx.request.url,
- 'title': title,
- 'content': content,
- }
- if extract_links:
- page_data['links'] = _extract_links(soup, ctx.request.url)
- if take_screenshot:
- screenshot_bytes = await page.screenshot(type='png')
- import base64
- page_data['screenshot'] = base64.b64encode(screenshot_bytes).decode()
-
- pages.append(page_data)
-
- # Follow links in crawl mode
- if mode == 'crawl':
- enqueue_kwargs: Dict[str, Any] = {'selector': link_selector}
- if url_pattern:
- import re
- from fnmatch import translate
- pattern = re.compile(translate(url_pattern))
- enqueue_kwargs['include'] = [pattern]
- await ctx.enqueue_links(**enqueue_kwargs)
-
- await asyncio.wait_for(
- crawler.run([url]),
- timeout=timeout_secs,
- )
diff --git a/server/services/handlers/tools.py b/server/services/handlers/tools.py
index 9779d52d..79033e12 100644
--- a/server/services/handlers/tools.py
+++ b/server/services/handlers/tools.py
@@ -186,18 +186,10 @@ async def execute_tool(tool_name: str, tool_args: Dict[str, Any],
if node_type == 'taskManager':
return await _execute_task_manager(tool_args, config)
- # Apify Actor (dual-purpose: workflow node + AI tool)
- if node_type == 'apifyActor':
- return await _execute_apify_actor(tool_args, config.get('parameters', {}))
-
# Browser automation (dual-purpose: workflow node + AI tool)
if node_type == 'browser':
return await _execute_browser(tool_args, config.get('parameters', {}), config)
- # Crawlee web scraper (dual-purpose: workflow node + AI tool)
- if node_type == 'crawleeScraper':
- return await _execute_crawlee_scraper(tool_args, config.get('parameters', {}))
-
# Proxy nodes (dual-purpose: workflow node + AI tool)
# Email nodes (dual-purpose: workflow node + AI tool)
if node_type in ('emailSend', 'emailRead'):
@@ -1193,78 +1185,6 @@ async def _execute_nearby_places(args: Dict[str, Any],
return {"error": f"Nearby places search failed: {str(e)}"}
-async def _execute_twitter_send(args: Dict[str, Any],
- node_params: Dict[str, Any]) -> Dict[str, Any]:
- """Execute Twitter send action via XDK SDK.
-
- Args:
- args: LLM-provided arguments (action, text, tweet_id, reply_to_id)
- node_params: Node parameters
-
- Returns:
- Twitter API result
- """
- from services.handlers.twitter import handle_twitter_send
-
- parameters = {**node_params, **args}
- # Handle reply_to_id alias
- if args.get('reply_to_id') and not args.get('tweet_id'):
- parameters['tweet_id'] = args['reply_to_id']
-
- return await handle_twitter_send(
- node_id="tool_twitter_send",
- node_type="twitterSend",
- parameters=parameters,
- context={}
- )
-
-
-async def _execute_twitter_search(args: Dict[str, Any],
- node_params: Dict[str, Any]) -> Dict[str, Any]:
- """Execute Twitter search via XDK SDK.
-
- Args:
- args: LLM-provided arguments (query, max_results)
- node_params: Node parameters
-
- Returns:
- Search results
- """
- from services.handlers.twitter import handle_twitter_search
-
- parameters = {**node_params, **args}
-
- return await handle_twitter_search(
- node_id="tool_twitter_search",
- node_type="twitterSearch",
- parameters=parameters,
- context={}
- )
-
-
-async def _execute_twitter_user(args: Dict[str, Any],
- node_params: Dict[str, Any]) -> Dict[str, Any]:
- """Execute Twitter user lookup via XDK SDK.
-
- Args:
- args: LLM-provided arguments (operation, username, user_id, max_results)
- node_params: Node parameters
-
- Returns:
- User data
- """
- from services.handlers.twitter import handle_twitter_user
-
- parameters = {**node_params, **args}
-
- return await handle_twitter_user(
- node_id="tool_twitter_user",
- node_type="twitterUser",
- parameters=parameters,
- context={}
- )
-
-
# _execute_google_gmail: deleted in Wave 11.C cleanup. gmail now routes
# through the plugin fast-path (nodes/google/gmail.py).
@@ -1274,42 +1194,6 @@ async def _execute_twitter_user(args: Dict[str, Any],
# (nodes/google/*.py). These functions were defined but unreferenced.
-async def _execute_apify_actor(args: Dict[str, Any],
- node_params: Dict[str, Any]) -> Dict[str, Any]:
- """Execute Apify actor via official SDK.
-
- Args:
- args: LLM-provided arguments (actor_id, input_json, max_results)
- node_params: Node parameters
-
- Returns:
- Apify actor run results
- """
- from services.handlers.apify import handle_apify_actor
-
- # Parse input_json string to dict if provided
- input_json = args.get('input_json', '{}')
- if isinstance(input_json, str):
- try:
- input_json = json.loads(input_json) if input_json.strip() else {}
- except json.JSONDecodeError:
- input_json = {}
-
- parameters = {
- **node_params,
- 'actorId': args.get('actor_id', node_params.get('actorId', '')),
- 'actorInput': input_json,
- 'maxResults': args.get('max_results', node_params.get('maxResults', 100)),
- }
-
- return await handle_apify_actor(
- node_id="tool_apify_actor",
- node_type="apifyActor",
- parameters=parameters,
- context={}
- )
-
-
async def _execute_browser(args: Dict[str, Any],
node_params: Dict[str, Any],
config: Dict[str, Any]) -> Dict[str, Any]:
@@ -1337,33 +1221,6 @@ async def _execute_browser(args: Dict[str, Any],
)
-async def _execute_crawlee_scraper(args: Dict[str, Any],
- node_params: Dict[str, Any]) -> Dict[str, Any]:
- """Execute Crawlee web scraper as an AI tool.
-
- Maps LLM-provided arguments to node parameters and delegates to handler.
- """
- from services.handlers.crawlee import handle_crawlee_scraper
-
- parameters = {
- **node_params,
- 'url': args.get('url', node_params.get('url', '')),
- 'crawlerType': args.get('crawlerType', node_params.get('crawlerType', 'beautifulsoup')),
- 'mode': args.get('mode', node_params.get('mode', 'single')),
- 'cssSelector': args.get('cssSelector', node_params.get('cssSelector', '')),
- 'maxPages': args.get('maxPages', node_params.get('maxPages', 10)),
- 'outputFormat': args.get('outputFormat', node_params.get('outputFormat', 'text')),
- 'useProxy': args.get('useProxy', node_params.get('useProxy', False)),
- }
-
- return await handle_crawlee_scraper(
- node_id="tool_crawlee_scraper",
- node_type="crawleeScraper",
- parameters=parameters,
- context={}
- )
-
-
async def _execute_proxy_request(args: Dict[str, Any],
node_params: Dict[str, Any],
config: Dict[str, Any]) -> Dict[str, Any]:
diff --git a/server/services/handlers/twitter.py b/server/services/handlers/twitter.py
deleted file mode 100644
index 962bdd0d..00000000
--- a/server/services/handlers/twitter.py
+++ /dev/null
@@ -1,652 +0,0 @@
-"""Twitter/X node handlers using official XDK SDK."""
-
-import asyncio
-import time
-from typing import Dict, Any, List
-from xdk import Client
-
-from core.logging import get_logger
-from services.pricing import get_pricing_service
-
-logger = get_logger(__name__)
-
-
-async def _track_twitter_usage(
- node_id: str,
- action: str,
- resource_count: int = 1,
- workflow_id: str = None,
- session_id: str = "default"
-) -> Dict[str, float]:
- """Track Twitter API usage for cost calculation.
-
- Args:
- node_id: The node executing the Twitter action
- action: Action name (tweet, search, followers, etc.)
- resource_count: Number of resources fetched (for paginated results)
- workflow_id: Optional workflow context
- session_id: Session for aggregation
-
- Returns:
- Cost breakdown dict with operation, unit_cost, resource_count, total_cost
- """
- from core.container import container
-
- pricing = get_pricing_service()
- cost_data = pricing.calculate_api_cost('twitter', action, resource_count)
-
- # Save to database
- db = container.database()
- await db.save_api_usage_metric({
- 'session_id': session_id,
- 'node_id': node_id,
- 'workflow_id': workflow_id,
- 'service': 'twitter',
- 'operation': cost_data.get('operation', action),
- 'endpoint': action,
- 'resource_count': resource_count,
- 'cost': cost_data.get('total_cost', 0.0)
- })
-
- logger.debug(f"[Twitter] Tracked usage: {action} x{resource_count} = ${cost_data.get('total_cost', 0):.6f}")
- return cost_data
-
-
-def _is_auth_error(e: Exception) -> bool:
- """Check if exception indicates an expired/invalid token."""
- s = str(e)
- return '401' in s or '403' in s or 'Unauthorized' in s or 'Forbidden' in s
-
-
-async def _get_twitter_client() -> Client:
- """Get authenticated Twitter client from stored OAuth 2.0 credentials.
-
- No validation call -- saves rate limits. If the token is expired,
- the actual API call will fail and the handler retries via _refresh_and_get_client().
- """
- from core.container import container
- auth_service = container.auth_service()
-
- tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")
- if not tokens or not tokens.get("access_token"):
- raise ValueError("Twitter not connected. Please authenticate via Credentials.")
-
- return Client(access_token=tokens["access_token"])
-
-
-async def _refresh_and_get_client() -> Client:
- """Refresh expired token and return new client."""
- from core.container import container
- auth_service = container.auth_service()
-
- tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")
- refresh_token = tokens.get("refresh_token", "") if tokens else ""
-
- if not refresh_token:
- raise ValueError("Twitter token expired. Please re-authenticate via Credentials.")
-
- from services.twitter_oauth import TwitterOAuth
-
- client_id = await auth_service.get_api_key("twitter_client_id") or ""
- client_secret = await auth_service.get_api_key("twitter_client_secret")
-
- oauth = TwitterOAuth(client_id=client_id, client_secret=client_secret, redirect_uri="")
- result = await oauth.refresh_access_token(refresh_token)
-
- if not result.get("success"):
- raise ValueError("Twitter token expired and refresh failed. Please re-authenticate.")
-
- new_access = result["access_token"]
- new_refresh = result.get("refresh_token", refresh_token)
-
- await auth_service.store_oauth_tokens(
- provider="twitter",
- access_token=new_access,
- refresh_token=new_refresh,
- email=tokens.get("email", ""),
- name=tokens.get("name", ""),
- scopes=tokens.get("scopes", ""),
- customer_id="owner",
- )
-
- logger.info("Twitter token refreshed successfully")
- return Client(access_token=new_access)
-
-
-async def _get_my_user_id(client: Client) -> str:
- """Get the authenticated user's ID."""
- response = await asyncio.to_thread(client.users.get_me)
- # response.data is a dict with 'id', 'username', 'name' keys
- return response.data["id"]
-
-
-# ---------------------------------------------------------------------------
-# Sync helper functions for asyncio.to_thread (XDK uses requests internally)
-# ---------------------------------------------------------------------------
-
-def _sync_create_post(client: Client, payload: dict):
- return client.posts.create(body=payload)
-
-
-def _sync_repost(client: Client, user_id: str, payload: dict):
- return client.users.repost_post(user_id, body=payload)
-
-
-def _sync_like(client: Client, user_id: str, payload: dict):
- return client.users.like_post(user_id, body=payload)
-
-
-def _sync_unlike(client: Client, user_id: str, tweet_id: str):
- return client.users.unlike_post(user_id, tweet_id=tweet_id)
-
-
-def _sync_delete_post(client: Client, tweet_id: str):
- return client.posts.delete(tweet_id)
-
-
-def _sync_search_recent(client: Client, query: str, max_results: int) -> dict:
- """Run search_recent synchronously, return first page data + includes.
-
- Returns dict with 'tweets' (raw tweet data) and 'includes' (expanded
- media/user/referenced_tweets objects) so callers can enrich output.
- """
- for page in client.posts.search_recent(
- query=query,
- max_results=max_results,
- tweet_fields=[
- "author_id", "created_at", "entities", "public_metrics",
- "possibly_sensitive", "lang", "source",
- "conversation_id", "in_reply_to_user_id", "referenced_tweets",
- "note_tweet",
- ],
- expansions=[
- "author_id",
- "attachments.media_keys",
- "referenced_tweets.id",
- "referenced_tweets.id.author_id",
- ],
- media_fields=["url", "preview_image_url", "type", "alt_text", "variants"],
- user_fields=["username", "name", "profile_image_url"],
- ):
- page_data = getattr(page, 'data', []) or []
- includes_raw = getattr(page, 'includes', None)
- includes_dict = {}
- if includes_raw:
- if hasattr(includes_raw, 'model_dump'):
- try:
- includes_dict = includes_raw.model_dump()
- except Exception:
- includes_dict = {}
- elif isinstance(includes_raw, dict):
- includes_dict = includes_raw
- return {"tweets": page_data, "includes": includes_dict}
- return {"tweets": [], "includes": {}}
-
-
-def _sync_get_me(client: Client, user_fields: list = None):
- return client.users.get_me(user_fields=user_fields)
-
-
-def _sync_get_by_usernames(client: Client, usernames: list, user_fields: list = None):
- return client.users.get_by_usernames(usernames=usernames, user_fields=user_fields)
-
-
-def _sync_get_by_ids(client: Client, ids: list, user_fields: list = None):
- return client.users.get_by_ids(ids=ids, user_fields=user_fields)
-
-
-def _sync_get_followers(client: Client, user_id: str, max_results: int) -> list:
- """Get first page of followers synchronously."""
- users = []
- for page in client.users.get_followers(
- user_id, max_results=max_results, user_fields=["created_at"]
- ):
- page_data = getattr(page, 'data', []) or []
- users.extend(page_data)
- break
- return users
-
-
-def _sync_get_following(client: Client, user_id: str, max_results: int) -> list:
- """Get first page of following synchronously."""
- users = []
- for page in client.users.get_following(
- user_id, max_results=max_results, user_fields=["created_at"]
- ):
- page_data = getattr(page, 'data', []) or []
- users.extend(page_data)
- break
- return users
-
-
-async def handle_twitter_send(
- node_id: str,
- node_type: str,
- parameters: Dict[str, Any],
- context: Dict[str, Any]
-) -> Dict[str, Any]:
- """Handle Twitter send actions: tweet, reply, retweet, like, unlike, delete."""
- start_time = time.time()
- action = parameters.get('action', 'tweet')
-
- try:
- client = await _get_twitter_client()
-
- # Extract workflow context for tracking
- workflow_id = context.get('workflow_id')
- session_id = context.get('session_id', 'default')
-
- try:
- return await _do_twitter_send(client, action, parameters, node_id, workflow_id, session_id, start_time)
- except Exception as e:
- if _is_auth_error(e):
- logger.info(f"Twitter auth error, refreshing token: {e}")
- client = await _refresh_and_get_client()
- return await _do_twitter_send(client, action, parameters, node_id, workflow_id, session_id, start_time)
- raise
-
- except Exception as e:
- logger.error(f"Twitter send error: {e}")
- return {"success": False, "error": str(e), "execution_time": time.time() - start_time}
-
-
-async def _do_twitter_send(
- client: Client, action: str, parameters: Dict[str, Any],
- node_id: str, workflow_id: str, session_id: str, start_time: float
-) -> Dict[str, Any]:
- """Execute a Twitter send action with the given client."""
- match action:
- case 'tweet':
- text = parameters.get('text', '')
- if not text:
- raise ValueError("Tweet text is required")
- payload = {"text": text[:280]}
- result = await asyncio.to_thread(_sync_create_post, client, payload)
- await _track_twitter_usage(node_id, 'tweet', 1, workflow_id, session_id)
- return _success(_format_response(result), "tweet_sent", start_time)
-
- case 'reply':
- text = parameters.get('text', '')
- reply_to = parameters.get('reply_to_id')
- if not text or not reply_to:
- raise ValueError("Text and reply_to_id are required")
- payload = {
- "text": text[:280],
- "reply": {"in_reply_to_tweet_id": reply_to}
- }
- result = await asyncio.to_thread(_sync_create_post, client, payload)
- await _track_twitter_usage(node_id, 'reply', 1, workflow_id, session_id)
- return _success(_format_response(result), "reply_sent", start_time)
-
- case 'retweet':
- tweet_id = parameters.get('tweet_id')
- if not tweet_id:
- raise ValueError("tweet_id is required")
- user_id = await _get_my_user_id(client)
- payload = {"tweet_id": tweet_id}
- result = await asyncio.to_thread(_sync_repost, client, user_id, payload)
- await _track_twitter_usage(node_id, 'retweet', 1, workflow_id, session_id)
- return _success(_format_response(result), "retweeted", start_time)
-
- case 'like':
- tweet_id = parameters.get('tweet_id')
- if not tweet_id:
- raise ValueError("tweet_id is required")
- user_id = await _get_my_user_id(client)
- payload = {"tweet_id": tweet_id}
- result = await asyncio.to_thread(_sync_like, client, user_id, payload)
- await _track_twitter_usage(node_id, 'like', 1, workflow_id, session_id)
- return _success(_format_response(result), "liked", start_time)
-
- case 'unlike':
- tweet_id = parameters.get('tweet_id')
- if not tweet_id:
- raise ValueError("tweet_id is required")
- user_id = await _get_my_user_id(client)
- result = await asyncio.to_thread(_sync_unlike, client, user_id, tweet_id)
- await _track_twitter_usage(node_id, 'unlike', 1, workflow_id, session_id)
- return _success(_format_response(result), "unliked", start_time)
-
- case 'delete':
- tweet_id = parameters.get('tweet_id')
- if not tweet_id:
- raise ValueError("tweet_id is required")
- result = await asyncio.to_thread(_sync_delete_post, client, tweet_id)
- await _track_twitter_usage(node_id, 'delete', 1, workflow_id, session_id)
- return _success(_format_response(result), "deleted", start_time)
-
- case _:
- raise ValueError(f"Unknown action: {action}")
-
-
-async def handle_twitter_search(
- node_id: str,
- node_type: str,
- parameters: Dict[str, Any],
- context: Dict[str, Any]
-) -> Dict[str, Any]:
- """Handle Twitter search operations."""
- start_time = time.time()
-
- try:
- client = await _get_twitter_client()
- query = parameters.get('query', '')
- # X API v2 requires max_results between 10 and 100 for search/recent
- max_results = max(10, min(parameters.get('max_results', 10), 100))
-
- if not query:
- raise ValueError("Search query is required")
-
- # Extract workflow context for tracking
- workflow_id = context.get('workflow_id')
- session_id = context.get('session_id', 'default')
-
- try:
- search_result = await asyncio.to_thread(_sync_search_recent, client, query, max_results)
- except Exception as e:
- if _is_auth_error(e):
- logger.info(f"Twitter auth error on search, refreshing token: {e}")
- client = await _refresh_and_get_client()
- search_result = await asyncio.to_thread(_sync_search_recent, client, query, max_results)
- else:
- raise
-
- raw_tweets = search_result["tweets"]
- includes = search_result["includes"]
-
- # Build lookup dicts from includes for enriching tweet data
- users_by_id = {}
- for u in (includes.get("users") or []):
- uid = u.get("id") if isinstance(u, dict) else getattr(u, "id", None)
- if uid:
- users_by_id[str(uid)] = u if isinstance(u, dict) else {
- k: v for k, v in u.__dict__.items() if not k.startswith('_')
- }
-
- media_by_key = {}
- for m in (includes.get("media") or []):
- mk = m.get("media_key") if isinstance(m, dict) else getattr(m, "media_key", None)
- if mk:
- media_by_key[mk] = m if isinstance(m, dict) else {
- k: v for k, v in m.__dict__.items() if not k.startswith('_')
- }
-
- tweets_by_id = {}
- for t in (includes.get("tweets") or []):
- tid = t.get("id") if isinstance(t, dict) else getattr(t, "id", None)
- if tid:
- tweets_by_id[str(tid)] = t if isinstance(t, dict) else {
- k: v for k, v in t.__dict__.items() if not k.startswith('_')
- }
-
- tweets = [_format_tweet(t, users_by_id, media_by_key, tweets_by_id) for t in raw_tweets]
-
- # Track: posts_read $0.005 per tweet fetched
- if tweets:
- await _track_twitter_usage(node_id, 'search', len(tweets), workflow_id, session_id)
-
- return {
- "success": True,
- "result": {"tweets": tweets, "count": len(tweets), "query": query},
- "execution_time": time.time() - start_time
- }
-
- except Exception as e:
- logger.error(f"Twitter search error: {e}")
- return {"success": False, "error": str(e), "execution_time": time.time() - start_time}
-
-
-async def handle_twitter_user(
- node_id: str,
- node_type: str,
- parameters: Dict[str, Any],
- context: Dict[str, Any]
-) -> Dict[str, Any]:
- """Handle Twitter user lookup operations."""
- start_time = time.time()
- operation = parameters.get('operation', 'me')
-
- try:
- client = await _get_twitter_client()
-
- # Extract workflow context for tracking
- workflow_id = context.get('workflow_id')
- session_id = context.get('session_id', 'default')
-
- try:
- return await _do_twitter_user(client, operation, parameters, node_id, workflow_id, session_id, start_time)
- except Exception as e:
- if _is_auth_error(e):
- logger.info(f"Twitter auth error on user op, refreshing token: {e}")
- client = await _refresh_and_get_client()
- return await _do_twitter_user(client, operation, parameters, node_id, workflow_id, session_id, start_time)
- raise
-
- except Exception as e:
- logger.error(f"Twitter user error: {e}")
- return {"success": False, "error": str(e), "execution_time": time.time() - start_time}
-
-
-async def _do_twitter_user(
- client: Client, operation: str, parameters: Dict[str, Any],
- node_id: str, workflow_id: str, session_id: str, start_time: float
-) -> Dict[str, Any]:
- """Execute a Twitter user operation with the given client."""
- match operation:
- case 'me':
- result = await asyncio.to_thread(_sync_get_me, client, ["created_at", "description"])
- await _track_twitter_usage(node_id, 'me', 1, workflow_id, session_id)
- return _success(_format_user_data(result.data), "user", start_time)
-
- case 'by_username':
- username = parameters.get('username')
- if not username:
- raise ValueError("Username is required")
- result = await asyncio.to_thread(
- _sync_get_by_usernames, client, [username], ["description", "created_at"]
- )
- users = getattr(result, 'data', []) or []
- if not users:
- raise ValueError(f"User @{username} not found")
- await _track_twitter_usage(node_id, 'by_username', 1, workflow_id, session_id)
- return _success(_format_user_data(users[0]), "user", start_time)
-
- case 'by_id':
- user_id = parameters.get('user_id')
- if not user_id:
- raise ValueError("User ID is required")
- result = await asyncio.to_thread(
- _sync_get_by_ids, client, [user_id], ["description", "created_at"]
- )
- users = getattr(result, 'data', []) or []
- if not users:
- raise ValueError(f"User ID {user_id} not found")
- await _track_twitter_usage(node_id, 'by_id', 1, workflow_id, session_id)
- return _success(_format_user_data(users[0]), "user", start_time)
-
- case 'followers':
- user_id = parameters.get('user_id')
- if not user_id:
- user_id = await _get_my_user_id(client)
- # X API requires max_results >= 1 for followers
- max_results = max(1, min(parameters.get('max_results', 100), 1000))
- raw_users = await asyncio.to_thread(_sync_get_followers, client, user_id, max_results)
- users = [_format_user_data(u) for u in raw_users]
- if users:
- await _track_twitter_usage(node_id, 'followers', len(users), workflow_id, session_id)
- return _success({"users": users, "count": len(users)}, "followers", start_time)
-
- case 'following':
- user_id = parameters.get('user_id')
- if not user_id:
- user_id = await _get_my_user_id(client)
- # X API requires max_results >= 1 for following
- max_results = max(1, min(parameters.get('max_results', 100), 1000))
- raw_users = await asyncio.to_thread(_sync_get_following, client, user_id, max_results)
- users = [_format_user_data(u) for u in raw_users]
- if users:
- await _track_twitter_usage(node_id, 'following', len(users), workflow_id, session_id)
- return _success({"users": users, "count": len(users)}, "following", start_time)
-
- case _:
- raise ValueError(f"Unknown operation: {operation}")
-
-
-def _success(data: Any, action: str, start_time: float) -> Dict[str, Any]:
- """Build success response."""
- return {
- "success": True,
- "result": data if isinstance(data, dict) else {"data": data, "action": action},
- "execution_time": time.time() - start_time
- }
-
-
-def _format_response(response) -> Dict[str, Any]:
- """Format XDK response object to dict."""
- if hasattr(response, 'data'):
- data = response.data
- if isinstance(data, dict):
- return data
- # Convert object attributes to dict
- return {k: v for k, v in data.__dict__.items() if not k.startswith('_')} if hasattr(data, '__dict__') else {"data": str(data)}
- return {"response": str(response)}
-
-
-def _format_tweet(tweet, users_by_id: Dict = None, media_by_key: Dict = None,
- tweets_by_id: Dict = None) -> Dict[str, Any]:
- """Format tweet object/dict to dict with enriched data from includes.
-
- Args:
- tweet: Raw tweet data (dict or object)
- users_by_id: {user_id: user_dict} from includes.users
- media_by_key: {media_key: media_dict} from includes.media
- tweets_by_id: {tweet_id: tweet_dict} from includes.tweets (referenced)
- """
- users_by_id = users_by_id or {}
- media_by_key = media_by_key or {}
- tweets_by_id = tweets_by_id or {}
-
- def _get(obj, key, default=None):
- return obj.get(key, default) if isinstance(obj, dict) else getattr(obj, key, default)
-
- tweet_id = _get(tweet, 'id')
- author_id = _get(tweet, 'author_id')
- text = _get(tweet, 'text', '')
-
- # Extract expanded URLs from entities to replace t.co links
- entities = _get(tweet, 'entities') or {}
- if isinstance(entities, dict):
- urls_list = entities.get('urls', [])
- elif hasattr(entities, 'urls'):
- urls_list = entities.urls or []
- else:
- urls_list = []
-
- expanded_urls = []
- display_text = text
- for u in urls_list:
- if isinstance(u, dict):
- short = u.get('url', '')
- expanded = u.get('expanded_url', '')
- display = u.get('display_url', '')
- else:
- short = getattr(u, 'url', '')
- expanded = getattr(u, 'expanded_url', '')
- display = getattr(u, 'display_url', '')
- if short and expanded:
- expanded_urls.append({"url": short, "expanded_url": expanded, "display_url": display})
- display_text = display_text.replace(short, expanded)
-
- # Note tweet (long-form tweets > 280 chars)
- note_tweet = _get(tweet, 'note_tweet')
- if note_tweet:
- note_text = note_tweet.get('text', '') if isinstance(note_tweet, dict) else getattr(note_tweet, 'text', '')
- if note_text:
- text = note_text
- display_text = note_text
-
- # Author info from includes
- author_info = users_by_id.get(str(author_id)) if author_id else None
-
- # Media from includes via attachments.media_keys
- attachments = _get(tweet, 'attachments') or {}
- if isinstance(attachments, dict):
- media_keys = attachments.get('media_keys', [])
- elif hasattr(attachments, 'media_keys'):
- media_keys = attachments.media_keys or []
- else:
- media_keys = []
- media_list = [media_by_key[k] for k in media_keys if k in media_by_key]
-
- # Referenced tweets (quoted, replied_to)
- ref_tweets_raw = _get(tweet, 'referenced_tweets') or []
- referenced = []
- for ref in ref_tweets_raw:
- if isinstance(ref, dict):
- ref_type = ref.get('type', '')
- ref_id = ref.get('id', '')
- else:
- ref_type = getattr(ref, 'type', '')
- ref_id = getattr(ref, 'id', '')
- ref_data = tweets_by_id.get(str(ref_id))
- referenced.append({
- "type": ref_type,
- "id": ref_id,
- "text": ref_data.get("text", "") if ref_data else None,
- "author_id": ref_data.get("author_id") if ref_data else None,
- })
-
- # Public metrics
- metrics = _get(tweet, 'public_metrics') or {}
- if not isinstance(metrics, dict) and hasattr(metrics, 'model_dump'):
- metrics = metrics.model_dump()
- elif not isinstance(metrics, dict):
- metrics = {}
-
- result = {
- "id": tweet_id,
- "text": text,
- "display_text": display_text,
- "author_id": author_id,
- "created_at": str(_get(tweet, 'created_at', '')),
- "lang": _get(tweet, 'lang'),
- "source": _get(tweet, 'source'),
- "conversation_id": _get(tweet, 'conversation_id'),
- "in_reply_to_user_id": _get(tweet, 'in_reply_to_user_id'),
- "possibly_sensitive": _get(tweet, 'possibly_sensitive', False),
- "public_metrics": metrics,
- }
-
- if author_info:
- result["author"] = author_info
- if expanded_urls:
- result["urls"] = expanded_urls
- if media_list:
- result["media"] = media_list
- if referenced:
- result["referenced_tweets"] = referenced
-
- return result
-
-
-def _format_user_data(user) -> Dict[str, Any]:
- """Format user object/dict to dict."""
- if isinstance(user, dict):
- return {
- "id": user.get("id"),
- "username": user.get("username"),
- "name": user.get("name"),
- "profile_image_url": user.get("profile_image_url"),
- "verified": user.get("verified", False),
- "description": user.get("description"),
- "created_at": str(user.get("created_at", "")),
- }
- return {
- "id": getattr(user, 'id', None),
- "username": getattr(user, 'username', None),
- "name": getattr(user, 'name', None),
- "profile_image_url": getattr(user, 'profile_image_url', None),
- "verified": getattr(user, 'verified', False),
- "description": getattr(user, 'description', None),
- "created_at": str(getattr(user, 'created_at', '')),
- }