Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
refactor(plugin): Wave 11.D.8a — twitter/crawlee/apify inlined
trohitg59bdb7e3bce2
diff --git a/server/nodes/scraper/apify_actor.py b/server/nodes/scraper/apify_actor.pyindex 1c6f20bc..f29eef07 100644--- a/server/nodes/scraper/apify_actor.py+++ b/server/nodes/scraper/apify_actor.py@@ -1,28 +1,133 @@-"""Apify Actor — Wave 11.C migration."""+"""Apify Actor — Wave 11.D.8 inlined.++Runs Apify actors (Instagram, TikTok, Twitter, LinkedIn, Google, Crawler)+via the official apify-client SDK. Quick-input helpers merge into the+raw ``actorInput`` JSON for common actors.+"""from __future__ import annotations-from typing import Any, Dict, Optional+import json+from typing import Any, Dict, List, Optionalfrom pydantic import BaseModel, ConfigDict, Field+from core.logging import get_loggerfrom services.plugin import ActionNode, NodeContext, Operation, TaskQueue+logger = get_logger(__name__)+++async def _get_apify_client():+ from apify_client import ApifyClientAsync # lazy — optional dep+ """Return an authenticated Apify client, or None if no token saved."""+ from core.container import container+ auth_service = container.auth_service()+ api_token = await auth_service.get_api_key("apify", "default")+ if not api_token:+ return None+ return ApifyClientAsync(api_token)+++def _build_actor_input(parameters: Dict[str, Any]) -> Dict[str, Any]:+ """Merge actor quick-input helpers into the raw ``actorInput`` JSON."""+ actor_id = parameters.get('actorId', '')+ if actor_id == 'custom':+ actor_id = parameters.get('customActorId', '')++ actor_input = parameters.get('actorInput', '{}')+ if isinstance(actor_input, str):+ try:+ actor_input = json.loads(actor_input) if actor_input.strip() else {}+ except json.JSONDecodeError:+ actor_input = {}+ elif not isinstance(actor_input, dict):+ actor_input = {}++ if actor_id == 'apify/instagram-scraper':+ urls = parameters.get('instagramUrls', '')+ if urls:+ actor_input['directUrls'] = [u.strip() for u in urls.split(',') if u.strip()]+ elif actor_id == 'clockworks/tiktok-scraper':+ profiles = parameters.get('tiktokProfiles', '')+ hashtags = parameters.get('tiktokHashtags', '')+ if profiles:+ actor_input['profiles'] = [p.strip() for p in profiles.split(',') if p.strip()]+ if hashtags:+ actor_input['hashtags'] = [h.strip() for h in hashtags.split(',') if h.strip()]+ elif actor_id == 'apidojo/tweet-scraper':+ search_terms = parameters.get('twitterSearchTerms', '')+ handles = parameters.get('twitterHandles', '')+ if search_terms:+ actor_input['searchTerms'] = [t.strip() for t in search_terms.split(',') if t.strip()]+ if handles:+ actor_input['twitterHandles'] = [h.strip() for h in handles.split(',') if h.strip()]+ elif actor_id == 'apify/google-search-scraper':+ query = parameters.get('googleSearchQuery', '')+ pages = parameters.get('googleSearchPages', 1)+ if query:+ actor_input['searchQuery'] = query+ actor_input['maxPagesPerQuery'] = pages+ elif actor_id == 'apify/website-content-crawler':+ start_urls = parameters.get('crawlerStartUrls', '')+ max_depth = parameters.get('crawlerMaxDepth', 2)+ max_pages = parameters.get('crawlerMaxPages', 50)+ if start_urls:+ actor_input['startUrls'] = [{'url': u.strip()} for u in start_urls.split(',') if u.strip()]+ actor_input['maxCrawlDepth'] = max_depth+ actor_input['maxCrawlPages'] = max_pages++ return actor_input+++async def validate_apify_token(api_token: str) -> Dict[str, Any]:+ """Validate an Apify API token by fetching /users/me.++ Used by the websocket ``validate_apify_key`` handler and by the+ Credentials modal. Lives on the plugin so there's no handler-file+ ghost left behind after Wave 11.D.8.+ """+ try:+ from apify_client import ApifyClientAsync # lazy — optional dep+ client = ApifyClientAsync(api_token)+ user_info = await client.user("me").get()+ if not user_info:+ return {"valid": False, "error": "Could not fetch user info"}+ plan = user_info.get("plan")+ return {+ "valid": True,+ "username": user_info.get("username", ""),+ "email": user_info.get("email", ""),+ "plan": plan.get("id", "free") if isinstance(plan, dict) else "free",+ }+ except Exception as e:+ logger.error(f"[Apify] Token validation error: {e}")+ msg = str(e)+ if "401" in msg or "Unauthorized" in msg:+ return {"valid": False, "error": "Invalid API token"}+ return {"valid": False, "error": msg}+class ApifyActorParams(BaseModel):- actor_id: str = Field(..., alias="actorId")- actor_input: Dict[str, Any] = Field(default_factory=dict, alias="actorInput")- max_results: int = Field(default=20, alias="maxResults", ge=1, le=10000)+ actor_id: str = Field(default="", alias="actorId")+ actor_input: Any = Field(default_factory=dict, alias="actorInput")+ max_results: int = Field(default=100, alias="maxResults", ge=1, le=10000)timeout: int = Field(default=300, ge=1, le=3600)- memory: int = Field(default=512, ge=128, le=8192)+ memory: int = Field(default=1024, ge=128, le=8192)model_config = ConfigDict(populate_by_name=True, extra="allow")class ApifyActorOutput(BaseModel):- items: Optional[list] = None- count: Optional[int] = Nonerun_id: Optional[str] = None+ actor_id: Optional[str] = None+ status: Optional[str] = None+ items: Optional[list] = None+ item_count: Optional[int] = None+ dataset_id: Optional[str] = None+ compute_units: Optional[float] = None+ started_at: Optional[str] = None+ finished_at: Optional[str] = Nonemodel_config = ConfigDict(extra="allow")@@ -48,12 +153,63 @@ class ApifyActorNode(ActionNode):Output = ApifyActorOutput@Operation("run")- async def run(self, ctx: NodeContext, params: ApifyActorParams) -> Any:- from services.handlers.apify import handle_apify_actor- response = await handle_apify_actor(- node_id=ctx.node_id, node_type=self.type,- parameters=params.model_dump(by_alias=True), context=ctx.raw,+ async def run(self, ctx: NodeContext, params: ApifyActorParams) -> ApifyActorOutput:+ client = await _get_apify_client()+ if not client:+ raise RuntimeError(+ "Apify API token not configured. Please add your token in Credentials.",+ )++ p = params.model_dump(by_alias=True)+ actor_id = p.get("actorId", "")+ if actor_id == "custom":+ actor_id = p.get("customActorId", "")+ if not actor_id:+ raise RuntimeError("Actor ID is required")++ actor_input = _build_actor_input(p)+ timeout_secs = int(p.get("timeout", 300))+ max_results = int(p.get("maxResults", 100))+ memory_mbytes = int(p.get("memory", 1024))++ logger.info(+ f"[Apify] Running actor {actor_id} "+ f"timeout={timeout_secs}s memory={memory_mbytes}MB",+ )+ run_info = await client.actor(actor_id).call(+ run_input=actor_input,+ timeout_secs=timeout_secs,+ memory_mbytes=memory_mbytes,+ )++ if run_info is None:+ raise RuntimeError("Actor run failed - no result returned")++ status = run_info.get("status", "UNKNOWN")+ run_id = run_info.get("id", "")+ dataset_id = run_info.get("defaultDatasetId", "")++ if status == "FAILED":+ raise RuntimeError(run_info.get("errorMessage", "Actor run failed"))+ if status == "TIMED-OUT":+ raise RuntimeError("Actor timed out. Try increasing the timeout.")+ if status == "ABORTED":+ raise RuntimeError("Actor run was aborted")++ items: List[Dict[str, Any]] = []+ if dataset_id:+ listing = await client.dataset(dataset_id).list_items(limit=max_results)+ items = listing.items if listing else []+ logger.info(f"[Apify] Actor {actor_id} completed: {len(items)} items")++ return ApifyActorOutput(+ run_id=run_id,+ actor_id=actor_id,+ status=status,+ items=items,+ item_count=len(items),+ dataset_id=dataset_id,+ compute_units=run_info.get("usageTotalUsd", 0),+ started_at=run_info.get("startedAt", ""),+ finished_at=run_info.get("finishedAt", ""),)- if response.get("success"):- return response.get("result") or response- raise RuntimeError(response.get("error") or "Apify run failed")diff --git a/server/nodes/scraper/crawlee_scraper.py b/server/nodes/scraper/crawlee_scraper.pyindex 25f757c4..e0bc574c 100644--- a/server/nodes/scraper/crawlee_scraper.py+++ b/server/nodes/scraper/crawlee_scraper.py@@ -1,16 +1,175 @@-"""Crawlee Scraper — Wave 11.C migration."""+"""Crawlee Scraper — Wave 11.D.8 inlined.++Thin wrapper over ``crawlee.BeautifulSoupCrawler`` (static HTML) and+``crawlee.PlaywrightCrawler`` (JS-rendered). Crawlee handles concurrency,+retries, storage, and anti-bot internally.+"""from __future__ import annotations-from typing import Any, Literal, Optional+import asyncio+from datetime import timedelta+from typing import Any, Dict, List, Literal, Optional+from urllib.parse import urljoinfrom pydantic import BaseModel, ConfigDict, Field+from core.logging import get_loggerfrom services.plugin import ActionNode, NodeContext, Operation, TaskQueue+logger = get_logger(__name__)++_MAX_CONTENT_LENGTH = 100_000+++async def _get_proxy_config(parameters: Dict[str, Any], url: str):+ """Bridge MachinaOs ProxyService to Crawlee ProxyConfiguration."""+ if not parameters.get('useProxy', False):+ return None+ try:+ from services.proxy import get_proxy_service+ proxy_svc = get_proxy_service()+ proxy_url = await proxy_svc.get_proxy_url(url, parameters)+ if not proxy_url:+ return None+ from crawlee.proxy_configuration import ProxyConfiguration+ return ProxyConfiguration(proxy_urls=[proxy_url])+ except Exception as e:+ logger.warning(f"[Crawlee] Proxy setup failed, proceeding without: {e}")+ return None+++def _extract_text(soup, css_selector: str, output_format: str) -> str:+ target = soup.select(css_selector) if css_selector else [soup]+ if output_format == 'html':+ parts = [str(el) for el in target]+ else:+ parts = [el.get_text(separator='\n', strip=True) for el in target]+ text = '\n\n'.join(parts)+ if output_format == 'markdown':+ try:+ import html2text+ h = html2text.HTML2Text()+ h.ignore_links = False+ h.ignore_images = False+ h.body_width = 0+ raw_html = '\n\n'.join(str(el) for el in target)+ text = h.handle(raw_html)+ except ImportError:+ pass+ return text[:_MAX_CONTENT_LENGTH]+++def _extract_links(soup, base_url: str) -> List[str]:+ return [urljoin(base_url, a['href']) for a in soup.find_all('a', href=True)]+++async def _run_beautifulsoup(url, pages, p, proxy_config, css_selector, extract_links,+ output_format, mode, max_pages, max_depth,+ max_concurrency, timeout_secs):+ from crawlee import ConcurrencySettings+ from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext+ from crawlee.storage_clients import MemoryStorageClient++ crawler = BeautifulSoupCrawler(+ max_requests_per_crawl=max_pages,+ max_crawl_depth=max_depth if mode == 'crawl' else 0,+ request_handler_timeout=timedelta(seconds=timeout_secs),+ concurrency_settings=ConcurrencySettings(+ max_concurrency=max_concurrency,+ desired_concurrency=min(max_concurrency, 10),+ ),+ storage_client=MemoryStorageClient(),+ proxy_configuration=proxy_config,+ configure_logging=False,+ )+ link_selector = p.get('linkSelector', '') or 'a[href]'+ url_pattern = p.get('urlPattern', '')++ @crawler.router.default_handler+ async def handler(ctx: BeautifulSoupCrawlingContext) -> None:+ title = ctx.soup.title.string if ctx.soup.title else ''+ content = _extract_text(ctx.soup, css_selector, output_format)+ page_data: Dict[str, Any] = {+ 'url': ctx.request.url, 'title': title, 'content': content,+ }+ if extract_links:+ page_data['links'] = _extract_links(ctx.soup, ctx.request.url)+ pages.append(page_data)++ if mode == 'crawl':+ kwargs: Dict[str, Any] = {'selector': link_selector}+ if url_pattern:+ import re+ from fnmatch import translate+ kwargs['include'] = [re.compile(translate(url_pattern))]+ await ctx.enqueue_links(**kwargs)++ await asyncio.wait_for(crawler.run([url]), timeout=timeout_secs)+++async def _run_playwright(url, pages, p, proxy_config, css_selector, extract_links,+ output_format, mode, max_pages, max_depth,+ max_concurrency, timeout_secs):+ from crawlee import ConcurrencySettings+ from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext+ from crawlee.storage_clients import MemoryStorageClient++ browser_type = p.get('browserType', 'chromium')+ wait_for_selector = p.get('waitForSelector', '')+ wait_timeout = p.get('waitTimeout', 30000)+ take_screenshot = p.get('screenshot', False) or p.get('takeScreenshot', False)++ crawler = PlaywrightCrawler(+ browser_type=browser_type, headless=True,+ max_requests_per_crawl=max_pages,+ max_crawl_depth=max_depth if mode == 'crawl' else 0,+ request_handler_timeout=timedelta(seconds=timeout_secs),+ concurrency_settings=ConcurrencySettings(+ max_concurrency=max_concurrency,+ desired_concurrency=min(max_concurrency, 10),+ ),+ storage_client=MemoryStorageClient(),+ proxy_configuration=proxy_config,+ configure_logging=False,+ )+ link_selector = p.get('linkSelector', '') or 'a[href]'+ url_pattern = p.get('urlPattern', '')++ @crawler.router.default_handler+ async def handler(ctx: PlaywrightCrawlingContext) -> None:+ page = ctx.page+ if wait_for_selector:+ await page.wait_for_selector(wait_for_selector, timeout=wait_timeout)+ title = await page.title()+ html = await page.content()+ from bs4 import BeautifulSoup+ soup = BeautifulSoup(html, 'html.parser')+ content = _extract_text(soup, css_selector, output_format)+ page_data: Dict[str, Any] = {+ 'url': ctx.request.url, 'title': title, 'content': content,+ }+ if extract_links:+ page_data['links'] = _extract_links(soup, ctx.request.url)+ if take_screenshot:+ import base64+ screenshot_bytes = await page.screenshot(type='png')+ page_data['screenshot'] = base64.b64encode(screenshot_bytes).decode()+ pages.append(page_data)++ if mode == 'crawl':+ kwargs: Dict[str, Any] = {'selector': link_selector}+ if url_pattern:+ import re+ from fnmatch import translate+ kwargs['include'] = [re.compile(translate(url_pattern))]+ await ctx.enqueue_links(**kwargs)++ await asyncio.wait_for(crawler.run([url]), timeout=timeout_secs)+class CrawleeScraperParams(BaseModel):- url: str = Field(...)+ url: str = Field(default="")crawler_type: Literal["beautifulsoup", "playwright", "adaptive"] = Field(default="beautifulsoup", alias="crawlerType",)@@ -19,13 +178,19 @@ class CrawleeScraperParams(BaseModel):extract_links: bool = Field(default=False, alias="extractLinks")max_pages: int = Field(default=10, alias="maxPages", ge=1, le=1000)take_screenshot: bool = Field(default=False, alias="takeScreenshot")+ output_format: Literal["text", "html", "markdown"] = Field(+ default="text", alias="outputFormat",+ )model_config = ConfigDict(populate_by_name=True, extra="allow")class CrawleeScraperOutput(BaseModel):- items: Optional[list] = None- pages_crawled: Optional[int] = None+ pages: Optional[list] = None+ page_count: Optional[int] = None+ crawler_type: Optional[str] = None+ mode: Optional[str] = None+ proxied: Optional[bool] = Nonemodel_config = ConfigDict(extra="allow")@@ -51,12 +216,55 @@ class CrawleeScraperNode(ActionNode):Output = CrawleeScraperOutput@Operation("scrape")- async def scrape(self, ctx: NodeContext, params: CrawleeScraperParams) -> Any:- from services.handlers.crawlee import handle_crawlee_scraper- response = await handle_crawlee_scraper(- node_id=ctx.node_id, node_type=self.type,- parameters=params.model_dump(by_alias=True), context=ctx.raw,+ async def scrape(self, ctx: NodeContext, params: CrawleeScraperParams) -> CrawleeScraperOutput:+ p = params.model_dump(by_alias=True)+ url = p.get('url', '').strip()+ if not url:+ raise RuntimeError("URL is required")++ crawler_type = p.get('crawlerType', 'beautifulsoup')+ mode = p.get('mode', 'single')+ css_selector = p.get('cssSelector', '')+ extract_links = p.get('extractLinks', False)+ output_format = p.get('outputFormat', 'text')+ max_pages = p.get('maxPages', 10) if mode == 'crawl' else 1+ max_depth = p.get('maxDepth', 2) if mode == 'crawl' else 0+ max_concurrency = p.get('maxConcurrency', 5)+ timeout_secs = p.get('timeout', 60)++ pages: List[Dict[str, Any]] = []+ proxy_config = await _get_proxy_config(p, url)++ try:+ if crawler_type in ('beautifulsoup', 'adaptive'):+ await _run_beautifulsoup(+ url, pages, p, proxy_config, css_selector, extract_links,+ output_format, mode, max_pages, max_depth,+ max_concurrency, timeout_secs,+ )+ elif crawler_type == 'playwright':+ await _run_playwright(+ url, pages, p, proxy_config, css_selector, extract_links,+ output_format, mode, max_pages, max_depth,+ max_concurrency, timeout_secs,+ )+ else:+ raise RuntimeError(f"Unknown crawler type: {crawler_type}")+ except ImportError as e:+ msg = str(e).lower()+ if 'playwright' in msg:+ raise RuntimeError(+ "Playwright not installed. Run: "+ "pip install 'crawlee[playwright]' && playwright install chromium",+ )+ if 'crawlee' in msg:+ raise RuntimeError(+ "Crawlee not installed. Run: pip install 'crawlee[beautifulsoup]'",+ )+ raise++ logger.info(f"[Crawlee] Scraped {len(pages)} page(s) from {url}")+ return CrawleeScraperOutput(+ pages=pages, page_count=len(pages),+ crawler_type=crawler_type, mode=mode, proxied=proxy_config is not None,)- if response.get("success"):- return response.get("result") or response- raise RuntimeError(response.get("error") or "Crawlee scrape failed")diff --git a/server/nodes/twitter/_base.py b/server/nodes/twitter/_base.pynew file mode 100644index 00000000..10ec08fb--- /dev/null+++ b/server/nodes/twitter/_base.py@@ -0,0 +1,311 @@+"""Twitter/X shared helpers (Wave 11.D.8).++Used by the four twitter plugins (send / search / user / receive).+Wraps the XDK SDK with:+- Lazy client auth (refresh on 401/403).+- asyncio.to_thread for all sync XDK calls.+- Pricing-service usage tracking.+- Response / tweet / user formatters.+"""++from __future__ import annotations++import asyncio+from typing import Any, Dict, List, Optional++from core.logging import get_logger+from services.pricing import get_pricing_service++logger = get_logger(__name__)+++def _is_auth_error(e: Exception) -> bool:+ s = str(e)+ return any(code in s for code in ("401", "403", "Unauthorized", "Forbidden"))+++async def track_twitter_usage(+ node_id: str, action: str, resource_count: int, context: Dict[str, Any],+) -> Dict[str, float]:+ """Record a Twitter API call in ``api_usage_metrics``."""+ from core.container import container++ pricing = get_pricing_service()+ cost_data = pricing.calculate_api_cost('twitter', action, resource_count)++ db = container.database()+ await db.save_api_usage_metric({+ 'session_id': context.get('session_id', 'default'),+ 'node_id': node_id,+ 'workflow_id': context.get('workflow_id'),+ 'service': 'twitter',+ 'operation': cost_data.get('operation', action),+ 'endpoint': action,+ 'resource_count': resource_count,+ 'cost': cost_data.get('total_cost', 0.0),+ })+ return cost_data+++async def get_twitter_client():+ """Build an XDK Client from the stored OAuth2 access token."""+ from core.container import container+ from xdk import Client++ auth_service = container.auth_service()+ tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")+ if not tokens or not tokens.get("access_token"):+ raise RuntimeError("Twitter not connected. Please authenticate via Credentials.")+ return Client(access_token=tokens["access_token"])+++async def refresh_and_get_client():+ """Refresh the OAuth2 token and return a new client."""+ from core.container import container+ from services.twitter_oauth import TwitterOAuth+ from xdk import Client++ auth_service = container.auth_service()+ tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")+ refresh_token = tokens.get("refresh_token", "") if tokens else ""+ if not refresh_token:+ raise RuntimeError("Twitter token expired. Please re-authenticate.")++ client_id = await auth_service.get_api_key("twitter_client_id") or ""+ client_secret = await auth_service.get_api_key("twitter_client_secret")++ oauth = TwitterOAuth(client_id=client_id, client_secret=client_secret, redirect_uri="")+ result = await oauth.refresh_access_token(refresh_token)+ if not result.get("success"):+ raise RuntimeError("Twitter token refresh failed. Please re-authenticate.")++ new_access = result["access_token"]+ new_refresh = result.get("refresh_token", refresh_token)++ await auth_service.store_oauth_tokens(+ provider="twitter",+ access_token=new_access,+ refresh_token=new_refresh,+ email=tokens.get("email", ""),+ name=tokens.get("name", ""),+ scopes=tokens.get("scopes", ""),+ customer_id="owner",+ )+ logger.info("Twitter token refreshed successfully")+ return Client(access_token=new_access)+++async def call_with_retry(fn, *args, **kwargs):+ """Run ``fn`` with an existing client; on auth failure, refresh and retry once.++ ``fn`` is an async callable taking the client as its first argument:+ ``async def fn(client, ...) -> Any``.+ """+ client = await get_twitter_client()+ try:+ return await fn(client, *args, **kwargs)+ except Exception as e:+ if _is_auth_error(e):+ logger.info(f"Twitter auth error, refreshing token: {e}")+ client = await refresh_and_get_client()+ return await fn(client, *args, **kwargs)+ raise+++async def get_my_user_id(client) -> str:+ resp = await asyncio.to_thread(client.users.get_me)+ return resp.data["id"]+++def format_response(response) -> Dict[str, Any]:+ if hasattr(response, 'data'):+ data = response.data+ if isinstance(data, dict):+ return data+ if hasattr(data, '__dict__'):+ return {k: v for k, v in data.__dict__.items() if not k.startswith('_')}+ return {"data": str(data)}+ return {"response": str(response)}+++def format_user(user) -> Dict[str, Any]:+ def g(attr, default=None):+ if isinstance(user, dict):+ return user.get(attr, default)+ return getattr(user, attr, default)+ return {+ "id": g("id"),+ "username": g("username"),+ "name": g("name"),+ "profile_image_url": g("profile_image_url"),+ "verified": g("verified", False),+ "description": g("description"),+ "created_at": str(g("created_at", "")),+ }+++def format_tweet(+ tweet,+ users_by_id: Optional[Dict] = None,+ media_by_key: Optional[Dict] = None,+ tweets_by_id: Optional[Dict] = None,+) -> Dict[str, Any]:+ """Format a tweet with enriched data from search includes."""+ users_by_id = users_by_id or {}+ media_by_key = media_by_key or {}+ tweets_by_id = tweets_by_id or {}++ def g(obj, key, default=None):+ return obj.get(key, default) if isinstance(obj, dict) else getattr(obj, key, default)++ tweet_id = g(tweet, 'id')+ author_id = g(tweet, 'author_id')+ text = g(tweet, 'text', '')++ entities = g(tweet, 'entities') or {}+ if isinstance(entities, dict):+ urls_list = entities.get('urls', [])+ elif hasattr(entities, 'urls'):+ urls_list = entities.urls or []+ else:+ urls_list = []++ expanded_urls = []+ display_text = text+ for u in urls_list:+ if isinstance(u, dict):+ short = u.get('url', ''); expanded = u.get('expanded_url', ''); display = u.get('display_url', '')+ else:+ short = getattr(u, 'url', ''); expanded = getattr(u, 'expanded_url', ''); display = getattr(u, 'display_url', '')+ if short and expanded:+ expanded_urls.append({"url": short, "expanded_url": expanded, "display_url": display})+ display_text = display_text.replace(short, expanded)++ note_tweet = g(tweet, 'note_tweet')+ if note_tweet:+ note_text = note_tweet.get('text', '') if isinstance(note_tweet, dict) else getattr(note_tweet, 'text', '')+ if note_text:+ text = note_text+ display_text = note_text++ author_info = users_by_id.get(str(author_id)) if author_id else None++ attachments = g(tweet, 'attachments') or {}+ if isinstance(attachments, dict):+ media_keys = attachments.get('media_keys', [])+ elif hasattr(attachments, 'media_keys'):+ media_keys = attachments.media_keys or []+ else:+ media_keys = []+ media_list = [media_by_key[k] for k in media_keys if k in media_by_key]++ ref_raw = g(tweet, 'referenced_tweets') or []+ referenced = []+ for ref in ref_raw:+ if isinstance(ref, dict):+ ref_type = ref.get('type', ''); ref_id = ref.get('id', '')+ else:+ ref_type = getattr(ref, 'type', ''); ref_id = getattr(ref, 'id', '')+ ref_data = tweets_by_id.get(str(ref_id))+ referenced.append({+ "type": ref_type,+ "id": ref_id,+ "text": ref_data.get("text", "") if ref_data else None,+ "author_id": ref_data.get("author_id") if ref_data else None,+ })++ metrics = g(tweet, 'public_metrics') or {}+ if not isinstance(metrics, dict) and hasattr(metrics, 'model_dump'):+ metrics = metrics.model_dump()+ elif not isinstance(metrics, dict):+ metrics = {}++ result = {+ "id": tweet_id,+ "text": text,+ "display_text": display_text,+ "author_id": author_id,+ "created_at": str(g(tweet, 'created_at', '')),+ "lang": g(tweet, 'lang'),+ "source": g(tweet, 'source'),+ "conversation_id": g(tweet, 'conversation_id'),+ "in_reply_to_user_id": g(tweet, 'in_reply_to_user_id'),+ "possibly_sensitive": g(tweet, 'possibly_sensitive', False),+ "public_metrics": metrics,+ }+ if author_info:+ result["author"] = author_info+ if expanded_urls:+ result["urls"] = expanded_urls+ if media_list:+ result["media"] = media_list+ if referenced:+ result["referenced_tweets"] = referenced+ return result+++def _includes_to_dict(raw) -> Dict[str, Any]:+ if not raw:+ return {}+ if hasattr(raw, 'model_dump'):+ try:+ return raw.model_dump()+ except Exception:+ return {}+ if isinstance(raw, dict):+ return raw+ return {}+++def sync_search_recent(client, query: str, max_results: int) -> Dict[str, Any]:+ """Run ``posts.search_recent`` synchronously; returns {tweets, includes}."""+ for page in client.posts.search_recent(+ query=query, max_results=max_results,+ tweet_fields=[+ "author_id", "created_at", "entities", "public_metrics",+ "possibly_sensitive", "lang", "source",+ "conversation_id", "in_reply_to_user_id", "referenced_tweets",+ "note_tweet",+ ],+ expansions=[+ "author_id", "attachments.media_keys",+ "referenced_tweets.id", "referenced_tweets.id.author_id",+ ],+ media_fields=["url", "preview_image_url", "type", "alt_text", "variants"],+ user_fields=["username", "name", "profile_image_url"],+ ):+ return {+ "tweets": getattr(page, 'data', []) or [],+ "includes": _includes_to_dict(getattr(page, 'includes', None)),+ }+ return {"tweets": [], "includes": {}}+++def includes_lookups(includes: Dict[str, Any]):+ """Build {users_by_id, media_by_key, tweets_by_id} from a search-includes dict."""+ def _as_dict(obj):+ if isinstance(obj, dict):+ return obj+ if hasattr(obj, '__dict__'):+ return {k: v for k, v in obj.__dict__.items() if not k.startswith('_')}+ return {}++ users_by_id = {}+ for u in (includes.get("users") or []):+ uid = u.get("id") if isinstance(u, dict) else getattr(u, "id", None)+ if uid:+ users_by_id[str(uid)] = _as_dict(u)++ media_by_key = {}+ for m in (includes.get("media") or []):+ mk = m.get("media_key") if isinstance(m, dict) else getattr(m, "media_key", None)+ if mk:+ media_by_key[mk] = _as_dict(m)++ tweets_by_id = {}+ for t in (includes.get("tweets") or []):+ tid = t.get("id") if isinstance(t, dict) else getattr(t, "id", None)+ if tid:+ tweets_by_id[str(tid)] = _as_dict(t)++ return users_by_id, media_by_key, tweets_by_iddiff --git a/server/nodes/twitter/twitter_search.py b/server/nodes/twitter/twitter_search.pyindex c92965d8..d25590e2 100644--- a/server/nodes/twitter/twitter_search.py+++ b/server/nodes/twitter/twitter_search.py@@ -1,16 +1,22 @@-"""Twitter Search — Wave 11.C migration."""+"""Twitter Search — Wave 11.D.8 inlined."""from __future__ import annotations-from typing import Any, Optional+import asyncio+from typing import Optionalfrom pydantic import BaseModel, ConfigDict, Fieldfrom services.plugin import ActionNode, NodeContext, Operation, TaskQueue+from ._base import (+ call_with_retry, format_tweet, includes_lookups,+ sync_search_recent, track_twitter_usage,+)+class TwitterSearchParams(BaseModel):- query: str = Field(..., min_length=1)+ query: str = Field(default="", min_length=0)max_results: int = Field(default=10, alias="maxResults", ge=10, le=100)model_config = ConfigDict(populate_by_name=True, extra="ignore")@@ -19,10 +25,20 @@ class TwitterSearchParams(BaseModel):class TwitterSearchOutput(BaseModel):tweets: Optional[list] = Nonecount: Optional[int] = None+ query: Optional[str] = Nonemodel_config = ConfigDict(extra="allow")+async def _do_search(client, query: str, max_results: int, node_id: str, ctx_raw: dict) -> TwitterSearchOutput:+ search = await asyncio.to_thread(sync_search_recent, client, query, max_results)+ users_by_id, media_by_key, tweets_by_id = includes_lookups(search["includes"])+ tweets = [format_tweet(t, users_by_id, media_by_key, tweets_by_id) for t in search["tweets"]]+ if tweets:+ await track_twitter_usage(node_id, "search", len(tweets), ctx_raw)+ return TwitterSearchOutput(tweets=tweets, count=len(tweets), query=query)++class TwitterSearchNode(ActionNode):type = "twitterSearch"display_name = "Twitter Search"@@ -46,12 +62,8 @@ class TwitterSearchNode(ActionNode):Output = TwitterSearchOutput@Operation("search", cost={"service": "twitter", "action": "search", "count": 1})- async def search(self, ctx: NodeContext, params: TwitterSearchParams) -> Any:- from services.handlers.twitter import handle_twitter_search- response = await handle_twitter_search(- node_id=ctx.node_id, node_type=self.type,- parameters=params.model_dump(by_alias=True), context=ctx.raw,- )- if response.get("success"):- return response.get("result") or response- raise RuntimeError(response.get("error") or "Twitter search failed")+ async def search(self, ctx: NodeContext, params: TwitterSearchParams) -> TwitterSearchOutput:+ if not params.query:+ raise RuntimeError("Search query is required")+ max_results = max(10, min(params.max_results, 100))+ return await call_with_retry(_do_search, params.query, max_results, ctx.node_id, ctx.raw)diff --git a/server/nodes/twitter/twitter_send.py b/server/nodes/twitter/twitter_send.pyindex 1d44e1b9..1846ce5b 100644--- a/server/nodes/twitter/twitter_send.py+++ b/server/nodes/twitter/twitter_send.py@@ -1,13 +1,18 @@-"""Twitter Send — Wave 11.C migration. Dual-purpose ActionNode + AI tool."""+"""Twitter Send — Wave 11.D.8 inlined (tweet/reply/retweet/like/unlike/delete)."""from __future__ import annotations-from typing import Any, Literal, Optional+import asyncio+from typing import Literal, Optionalfrom pydantic import BaseModel, ConfigDict, Fieldfrom services.plugin import ActionNode, NodeContext, Operation, TaskQueue+from ._base import (+ call_with_retry, format_response, get_my_user_id, track_twitter_usage,+)+class TwitterSendParams(BaseModel):action: Literal["tweet", "reply", "retweet", "like", "unlike", "delete"] = "tweet"@@ -19,12 +24,69 @@ class TwitterSendParams(BaseModel):class TwitterSendOutput(BaseModel):- tweet_id: Optional[str] = None- success: Optional[bool] = None+ action: Optional[str] = None+ data: Optional[dict] = Nonemodel_config = ConfigDict(extra="allow")+async def _do_send(client, action: str, p: dict, node_id: str, ctx_raw: dict) -> TwitterSendOutput:+ if action == "tweet":+ text = p.get("text", "")+ if not text:+ raise RuntimeError("Tweet text is required")+ result = await asyncio.to_thread(client.posts.create, body={"text": text[:280]})+ await track_twitter_usage(node_id, "tweet", 1, ctx_raw)+ return TwitterSendOutput(action="tweet_sent", data=format_response(result))++ if action == "reply":+ text = p.get("text", "")+ reply_to = p.get("replyToId") or p.get("reply_to_id")+ if not text or not reply_to:+ raise RuntimeError("Text and reply_to_id are required")+ result = await asyncio.to_thread(+ client.posts.create,+ body={"text": text[:280], "reply": {"in_reply_to_tweet_id": reply_to}},+ )+ await track_twitter_usage(node_id, "reply", 1, ctx_raw)+ return TwitterSendOutput(action="reply_sent", data=format_response(result))++ tweet_id = p.get("tweetId") or p.get("tweet_id")+ if not tweet_id:+ raise RuntimeError("tweet_id is required")++ if action == "retweet":+ user_id = await get_my_user_id(client)+ result = await asyncio.to_thread(+ client.users.repost_post, user_id, body={"tweet_id": tweet_id},+ )+ await track_twitter_usage(node_id, "retweet", 1, ctx_raw)+ return TwitterSendOutput(action="retweeted", data=format_response(result))++ if action == "like":+ user_id = await get_my_user_id(client)+ result = await asyncio.to_thread(+ client.users.like_post, user_id, body={"tweet_id": tweet_id},+ )+ await track_twitter_usage(node_id, "like", 1, ctx_raw)+ return TwitterSendOutput(action="liked", data=format_response(result))++ if action == "unlike":+ user_id = await get_my_user_id(client)+ result = await asyncio.to_thread(+ client.users.unlike_post, user_id, tweet_id=tweet_id,+ )+ await track_twitter_usage(node_id, "unlike", 1, ctx_raw)+ return TwitterSendOutput(action="unliked", data=format_response(result))++ if action == "delete":+ result = await asyncio.to_thread(client.posts.delete, tweet_id)+ await track_twitter_usage(node_id, "delete", 1, ctx_raw)+ return TwitterSendOutput(action="deleted", data=format_response(result))++ raise RuntimeError(f"Unknown action: {action}")++class TwitterSendNode(ActionNode):type = "twitterSend"display_name = "Twitter Send"@@ -48,12 +110,7 @@ class TwitterSendNode(ActionNode):Output = TwitterSendOutput@Operation("send", cost={"service": "twitter", "action": "send", "count": 1})- async def send(self, ctx: NodeContext, params: TwitterSendParams) -> Any:- from services.handlers.twitter import handle_twitter_send- response = await handle_twitter_send(- node_id=ctx.node_id, node_type=self.type,- parameters=params.model_dump(by_alias=True), context=ctx.raw,- )- if response.get("success"):- return response.get("result") or response- raise RuntimeError(response.get("error") or "Twitter send failed")+ async def send(self, ctx: NodeContext, params: TwitterSendParams) -> TwitterSendOutput:+ p = params.model_dump(by_alias=True)+ action = p.get("action", "tweet")+ return await call_with_retry(_do_send, action, p, ctx.node_id, ctx.raw)diff --git a/server/nodes/twitter/twitter_user.py b/server/nodes/twitter/twitter_user.pyindex d16fe4bf..bb0d1655 100644--- a/server/nodes/twitter/twitter_user.py+++ b/server/nodes/twitter/twitter_user.py@@ -1,13 +1,16 @@-"""Twitter User — Wave 11.C migration."""+"""Twitter User — Wave 11.D.8 inlined (me / by_username / by_id / followers / following)."""from __future__ import annotations-from typing import Any, Literal, Optional+import asyncio+from typing import Literal, Optionalfrom pydantic import BaseModel, ConfigDict, Fieldfrom services.plugin import ActionNode, NodeContext, Operation, TaskQueue+from ._base import call_with_retry, format_user, get_my_user_id, track_twitter_usage+class TwitterUserParams(BaseModel):operation: Literal["me", "by_username", "by_id", "followers", "following"] = "me"@@ -19,12 +22,79 @@ class TwitterUserParams(BaseModel):class TwitterUserOutput(BaseModel):+ operation: Optional[str] = Noneuser: Optional[dict] = Noneusers: Optional[list] = None+ count: Optional[int] = Nonemodel_config = ConfigDict(extra="allow")+def _sync_followers(client, user_id, max_results):+ for page in client.users.get_followers(+ user_id, max_results=max_results, user_fields=["created_at"],+ ):+ return list(getattr(page, 'data', []) or [])+ return []+++def _sync_following(client, user_id, max_results):+ for page in client.users.get_following(+ user_id, max_results=max_results, user_fields=["created_at"],+ ):+ return list(getattr(page, 'data', []) or [])+ return []+++async def _do_lookup(client, op: str, p: dict, node_id: str, ctx_raw: dict) -> TwitterUserOutput:+ if op == "me":+ result = await asyncio.to_thread(+ client.users.get_me, user_fields=["created_at", "description"],+ )+ await track_twitter_usage(node_id, "me", 1, ctx_raw)+ return TwitterUserOutput(operation="me", user=format_user(result.data))++ if op == "by_username":+ username = p.get("username")+ if not username:+ raise RuntimeError("Username is required")+ result = await asyncio.to_thread(+ client.users.get_by_usernames, usernames=[username],+ user_fields=["description", "created_at"],+ )+ users = getattr(result, 'data', []) or []+ if not users:+ raise RuntimeError(f"User @{username} not found")+ await track_twitter_usage(node_id, "by_username", 1, ctx_raw)+ return TwitterUserOutput(operation="by_username", user=format_user(users[0]))++ if op == "by_id":+ user_id = p.get("userId") or p.get("user_id")+ if not user_id:+ raise RuntimeError("User ID is required")+ result = await asyncio.to_thread(+ client.users.get_by_ids, ids=[user_id],+ user_fields=["description", "created_at"],+ )+ users = getattr(result, 'data', []) or []+ if not users:+ raise RuntimeError(f"User ID {user_id} not found")+ await track_twitter_usage(node_id, "by_id", 1, ctx_raw)+ return TwitterUserOutput(operation="by_id", user=format_user(users[0]))++ if op in ("followers", "following"):+ user_id = p.get("userId") or p.get("user_id") or await get_my_user_id(client)+ max_results = max(1, min(int(p.get("maxResults", 100)), 1000))+ fetch = _sync_followers if op == "followers" else _sync_following+ raw = await asyncio.to_thread(fetch, client, user_id, max_results)+ users = [format_user(u) for u in raw]+ if users:+ await track_twitter_usage(node_id, op, len(users), ctx_raw)+ return TwitterUserOutput(operation=op, users=users, count=len(users))++ raise RuntimeError(f"Unknown operation: {op}")++class TwitterUserNode(ActionNode):type = "twitterUser"display_name = "Twitter User"@@ -48,12 +118,7 @@ class TwitterUserNode(ActionNode):Output = TwitterUserOutput@Operation("lookup", cost={"service": "twitter", "action": "user_lookup", "count": 1})- async def lookup(self, ctx: NodeContext, params: TwitterUserParams) -> Any:- from services.handlers.twitter import handle_twitter_user- response = await handle_twitter_user(- node_id=ctx.node_id, node_type=self.type,- parameters=params.model_dump(by_alias=True), context=ctx.raw,- )- if response.get("success"):- return response.get("result") or response- raise RuntimeError(response.get("error") or "Twitter user lookup failed")+ async def lookup(self, ctx: NodeContext, params: TwitterUserParams) -> TwitterUserOutput:+ p = params.model_dump(by_alias=True)+ op = p.get("operation", "me")+ return await call_with_retry(_do_lookup, op, p, ctx.node_id, ctx.raw)diff --git a/server/routers/websocket.py b/server/routers/websocket.pyindex 57b93738..29d7b3f9 100644--- a/server/routers/websocket.py+++ b/server/routers/websocket.py@@ -1810,7 +1810,7 @@ async def handle_validate_maps_key(data: Dict[str, Any], websocket: WebSocket) -async def handle_validate_apify_key(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:"""Validate Apify API token and save to database if valid."""- from services.handlers.apify import validate_apify_token+ from nodes.scraper.apify_actor import validate_apify_tokenbroadcaster = get_status_broadcaster()auth_service = container.auth_service()diff --git a/server/services/handlers/__init__.py b/server/services/handlers/__init__.pyindex cad2cdb3..d677e20b 100644--- a/server/services/handlers/__init__.py+++ b/server/services/handlers/__init__.py@@ -72,12 +72,7 @@ from .whatsapp import (handle_whatsapp_db,)-# Twitter handlers-from .twitter import (- handle_twitter_send,- handle_twitter_search,- handle_twitter_user,-)+# Twitter handlers inlined into nodes/twitter/*.py (Wave 11.D.8).# Google Workspace handlers inlined into nodes/google/*.py (Wave 11.D.4).@@ -99,8 +94,7 @@ from .search import (# Email handlers (Himalaya CLI)from .email import handle_email_send, handle_email_read, handle_email_receive-# Crawlee web scraping handler-from .crawlee import handle_crawlee_scraper+# Crawlee handler inlined into nodes/scraper/crawlee_scraper.py (Wave 11.D.8).# Browser automation handler (agent-browser CLI)from .browser import handle_browser@@ -165,9 +159,6 @@ __all__ = ['handle_whatsapp_send','handle_whatsapp_db',- 'handle_twitter_send',- 'handle_twitter_search',- 'handle_twitter_user',# Social'handle_social_receive','handle_social_send',@@ -175,8 +166,6 @@ __all__ = ['handle_brave_search','handle_serper_search','handle_perplexity_search',- # Crawlee- 'handle_crawlee_scraper',# RLM'handle_rlm_agent',# Deep Agentdiff --git a/server/services/handlers/apify.py b/server/services/handlers/apify.pydeleted file mode 100644index 700c755f..00000000--- a/server/services/handlers/apify.py+++ /dev/null@@ -1,261 +0,0 @@-"""Apify node handlers using official apify-client SDK."""--import json-import time-from typing import Dict, Any, Optional, List--from apify_client import ApifyClientAsync--from core.logging import get_logger--logger = get_logger(__name__)---async def _get_apify_client(context: Dict[str, Any]) -> Optional[ApifyClientAsync]:- """Get authenticated Apify client from stored credentials."""- from core.container import container- auth_service = container.auth_service()- api_token = await auth_service.get_api_key("apify", "default")- if not api_token:- return None- return ApifyClientAsync(api_token)---def _build_actor_input(parameters: Dict[str, Any]) -> Dict[str, Any]:- """Build actor input from parameters, merging quick helpers with raw JSON input."""- actor_id = parameters.get('actorId', '')- if actor_id == 'custom':- actor_id = parameters.get('customActorId', '')-- # Start with raw JSON input if provided- actor_input = parameters.get('actorInput', '{}')- if isinstance(actor_input, str):- try:- actor_input = json.loads(actor_input) if actor_input.strip() else {}- except json.JSONDecodeError:- actor_input = {}- elif not isinstance(actor_input, dict):- actor_input = {}-- # Merge quick helpers based on actor type- if actor_id == 'apify/instagram-scraper':- urls = parameters.get('instagramUrls', '')- if urls:- actor_input['directUrls'] = [u.strip() for u in urls.split(',') if u.strip()]-- elif actor_id == 'clockworks/tiktok-scraper':- profiles = parameters.get('tiktokProfiles', '')- hashtags = parameters.get('tiktokHashtags', '')- if profiles:- actor_input['profiles'] = [p.strip() for p in profiles.split(',') if p.strip()]- if hashtags:- actor_input['hashtags'] = [h.strip() for h in hashtags.split(',') if h.strip()]-- elif actor_id == 'apidojo/tweet-scraper':- search_terms = parameters.get('twitterSearchTerms', '')- handles = parameters.get('twitterHandles', '')- if search_terms:- actor_input['searchTerms'] = [t.strip() for t in search_terms.split(',') if t.strip()]- if handles:- actor_input['twitterHandles'] = [h.strip() for h in handles.split(',') if h.strip()]-- elif actor_id == 'apify/google-search-scraper':- query = parameters.get('googleSearchQuery', '')- pages = parameters.get('googleSearchPages', 1)- if query:- actor_input['searchQuery'] = query- actor_input['maxPagesPerQuery'] = pages-- elif actor_id == 'apify/website-content-crawler':- start_urls = parameters.get('crawlerStartUrls', '')- max_depth = parameters.get('crawlerMaxDepth', 2)- max_pages = parameters.get('crawlerMaxPages', 50)- if start_urls:- actor_input['startUrls'] = [{'url': u.strip()} for u in start_urls.split(',') if u.strip()]- actor_input['maxCrawlDepth'] = max_depth- actor_input['maxCrawlPages'] = max_pages-- return actor_input---async def handle_apify_actor(- node_id: str,- node_type: str,- parameters: Dict[str, Any],- context: Dict[str, Any]-) -> Dict[str, Any]:- """Run an Apify actor and return results using official SDK.-- Args:- node_id: Node ID- node_type: Node type ('apifyActor')- parameters: Node parameters including actorId, actorInput, maxResults, timeout- context: Execution context-- Returns:- Dict with success status, results, and execution metadata- """- start_time = time.time()-- try:- # Get authenticated client- client = await _get_apify_client(context)- if not client:- return {- "success": False,- "error": "Apify API token not configured. Please add your token in Credentials.",- "execution_time": time.time() - start_time- }-- # Get actor ID- actor_id = parameters.get("actorId", "")- if actor_id == "custom":- actor_id = parameters.get("customActorId", "")- if not actor_id:- return {- "success": False,- "error": "Actor ID is required",- "execution_time": time.time() - start_time- }-- # Build actor input- actor_input = _build_actor_input(parameters)-- # Get execution options- timeout_secs = parameters.get("timeout", 300)- max_results = parameters.get("maxResults", 100)- memory_mbytes = parameters.get("memory", 1024)-- logger.info(f"[Apify] Running actor {actor_id} with timeout={timeout_secs}s, memory={memory_mbytes}MB")-- # Get actor client and run- actor_client = client.actor(actor_id)-- # call() runs the actor and waits for it to finish- # SDK handles polling and retries automatically- run_info = await actor_client.call(- run_input=actor_input,- timeout_secs=timeout_secs,- memory_mbytes=memory_mbytes- )-- if run_info is None:- return {- "success": False,- "error": "Actor run failed - no result returned",- "execution_time": time.time() - start_time- }-- run_status = run_info.get("status", "UNKNOWN")- run_id = run_info.get("id", "")- dataset_id = run_info.get("defaultDatasetId", "")-- # Handle non-success statuses- if run_status == "FAILED":- error_msg = run_info.get("errorMessage", "Actor run failed")- logger.error(f"[Apify] Actor {actor_id} failed: {error_msg}")- return {- "success": False,- "error": error_msg,- "result": {- "run_id": run_id,- "actor_id": actor_id,- "status": run_status- },- "execution_time": time.time() - start_time- }- elif run_status == "TIMED-OUT":- logger.warning(f"[Apify] Actor {actor_id} timed out")- return {- "success": False,- "error": "Actor timed out. Try increasing the timeout.",- "result": {- "run_id": run_id,- "actor_id": actor_id,- "status": run_status- },- "execution_time": time.time() - start_time- }- elif run_status == "ABORTED":- return {- "success": False,- "error": "Actor run was aborted",- "result": {- "run_id": run_id,- "actor_id": actor_id,- "status": run_status- },- "execution_time": time.time() - start_time- }-- # Get results from dataset- items: List[Dict[str, Any]] = []- if dataset_id:- dataset_client = client.dataset(dataset_id)- list_result = await dataset_client.list_items(limit=max_results)- items = list_result.items if list_result else []- logger.info(f"[Apify] Actor {actor_id} completed: {len(items)} items retrieved")-- # Build result- result = {- "run_id": run_id,- "actor_id": actor_id,- "status": run_status,- "items": items,- "item_count": len(items),- "dataset_id": dataset_id,- "compute_units": run_info.get("usageTotalUsd", 0),- "started_at": run_info.get("startedAt", ""),- "finished_at": run_info.get("finishedAt", "")- }-- return {- "success": True,- "result": result,- "execution_time": time.time() - start_time- }-- except Exception as e:- logger.error(f"[Apify] Error: {e}")- error_msg = str(e)- # Improve error messages for common cases- if "401" in error_msg or "Unauthorized" in error_msg:- error_msg = "Invalid Apify API token. Please check your credentials."- elif "404" in error_msg or "not found" in error_msg.lower():- error_msg = "Actor not found. Check the actor ID."- return {- "success": False,- "error": error_msg,- "execution_time": time.time() - start_time- }---async def validate_apify_token(api_token: str) -> Dict[str, Any]:- """Validate Apify API token by fetching user info.-- Args:- api_token: Apify API token to validate-- Returns:- Dict with 'valid' boolean and user info if valid- """- try:- client = ApifyClientAsync(api_token)- user_client = client.user("me")- user_info = await user_client.get()-- if user_info:- return {- "valid": True,- "username": user_info.get("username", ""),- "email": user_info.get("email", ""),- "plan": user_info.get("plan", {}).get("id", "free") if isinstance(user_info.get("plan"), dict) else "free"- }- return {"valid": False, "error": "Could not fetch user info"}-- except Exception as e:- logger.error(f"[Apify] Token validation error: {e}")- error_msg = str(e)- if "401" in error_msg or "Unauthorized" in error_msg:- return {"valid": False, "error": "Invalid API token"}- return {"valid": False, "error": str(e)}diff --git a/server/services/handlers/crawlee.py b/server/services/handlers/crawlee.pydeleted file mode 100644index 6f359d4a..00000000--- a/server/services/handlers/crawlee.py+++ /dev/null@@ -1,324 +0,0 @@-"""Crawlee web scraping handler using the crawlee library.--Thin wrapper around Crawlee's BeautifulSoupCrawler and PlaywrightCrawler.-The library handles concurrency, retries, storage, and anti-bot internally.-"""--import asyncio-import time-from datetime import timedelta-from typing import Any, Dict, List, Optional--from core.logging import get_logger--logger = get_logger(__name__)--# Max content size per page to prevent memory issues-_MAX_CONTENT_LENGTH = 100_000---async def _get_proxy_config(parameters: Dict[str, Any], url: str):- """Bridge MachinaOs ProxyService to Crawlee ProxyConfiguration.-- Returns a crawlee.ProxyConfiguration or None.- """- if not parameters.get('useProxy', False):- return None-- try:- from services.proxy import get_proxy_service- proxy_svc = get_proxy_service()- proxy_url = await proxy_svc.get_proxy_url(url, parameters)- if not proxy_url:- return None-- from crawlee.proxy_configuration import ProxyConfiguration- return ProxyConfiguration(proxy_urls=[proxy_url])- except Exception as e:- logger.warning(f"[Crawlee] Proxy setup failed, proceeding without: {e}")- return None---def _extract_text(soup, css_selector: str, output_format: str) -> str:- """Extract content from BeautifulSoup object."""- target = soup.select(css_selector) if css_selector else [soup]-- if output_format == 'html':- parts = [str(el) for el in target]- else:- parts = [el.get_text(separator='\n', strip=True) for el in target]-- text = '\n\n'.join(parts)-- if output_format == 'markdown':- try:- import html2text- h = html2text.HTML2Text()- h.ignore_links = False- h.ignore_images = False- h.body_width = 0- raw_html = '\n\n'.join(str(el) for el in target)- text = h.handle(raw_html)- except ImportError:- pass # Fall back to plain text-- return text[:_MAX_CONTENT_LENGTH]---def _extract_links(soup, base_url: str) -> List[str]:- """Extract all links from page."""- from urllib.parse import urljoin- links = []- for a in soup.find_all('a', href=True):- links.append(urljoin(base_url, a['href']))- return links---async def handle_crawlee_scraper(- node_id: str,- node_type: str,- parameters: Dict[str, Any],- context: Dict[str, Any]-) -> Dict[str, Any]:- """Scrape web pages using Crawlee (BeautifulSoup or Playwright).-- Args:- node_id: Node ID- node_type: Node type ('crawleeScraper')- parameters: Node parameters- context: Execution context-- Returns:- Dict with success status, scraped pages, and metadata- """- start_time = time.time()-- url = parameters.get('url', '').strip()- if not url:- return {- "success": False,- "error": "URL is required",- "execution_time": time.time() - start_time- }-- crawler_type = parameters.get('crawlerType', 'beautifulsoup')- mode = parameters.get('mode', 'single')- css_selector = parameters.get('cssSelector', '')- extract_links = parameters.get('extractLinks', False)- output_format = parameters.get('outputFormat', 'text')- max_pages = parameters.get('maxPages', 10) if mode == 'crawl' else 1- max_depth = parameters.get('maxDepth', 2) if mode == 'crawl' else 0- max_concurrency = parameters.get('maxConcurrency', 5)- timeout_secs = parameters.get('timeout', 60)-- # Collect results from crawler handlers- pages: List[Dict[str, Any]] = []-- try:- proxy_config = await _get_proxy_config(parameters, url)-- if crawler_type in ('beautifulsoup', 'adaptive'):- await _run_beautifulsoup(- url, pages, parameters, proxy_config,- css_selector, extract_links, output_format,- mode, max_pages, max_depth, max_concurrency, timeout_secs,- )- elif crawler_type == 'playwright':- await _run_playwright(- url, pages, parameters, proxy_config,- css_selector, extract_links, output_format,- mode, max_pages, max_depth, max_concurrency, timeout_secs,- )- else:- return {- "success": False,- "error": f"Unknown crawler type: {crawler_type}",- "execution_time": time.time() - start_time- }-- logger.info(f"[Crawlee] Scraped {len(pages)} page(s) from {url}")-- return {- "success": True,- "result": {- "pages": pages,- "page_count": len(pages),- "crawler_type": crawler_type,- "mode": mode,- "proxied": proxy_config is not None,- },- "execution_time": time.time() - start_time- }-- except ImportError as e:- msg = str(e)- if 'playwright' in msg.lower():- msg = ("Playwright not installed. Run: pip install 'crawlee[playwright]' && playwright install chromium")- elif 'crawlee' in msg.lower():- msg = ("Crawlee not installed. Run: pip install 'crawlee[beautifulsoup]'")- return {- "success": False,- "error": msg,- "execution_time": time.time() - start_time- }- except Exception as e:- logger.error(f"[Crawlee] Error scraping {url}: {e}")- return {- "success": False,- "error": str(e),- "execution_time": time.time() - start_time- }---async def _run_beautifulsoup(- url: str,- pages: List[Dict],- parameters: Dict[str, Any],- proxy_config,- css_selector: str,- extract_links: bool,- output_format: str,- mode: str,- max_pages: int,- max_depth: int,- max_concurrency: int,- timeout_secs: int,-) -> None:- """Run BeautifulSoupCrawler and collect results into pages list."""- from crawlee import ConcurrencySettings- from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext- from crawlee.storage_clients import MemoryStorageClient-- crawler = BeautifulSoupCrawler(- max_requests_per_crawl=max_pages,- max_crawl_depth=max_depth if mode == 'crawl' else 0,- request_handler_timeout=timedelta(seconds=timeout_secs),- concurrency_settings=ConcurrencySettings(- max_concurrency=max_concurrency,- desired_concurrency=min(max_concurrency, 10),- ),- storage_client=MemoryStorageClient(),- proxy_configuration=proxy_config,- configure_logging=False,- )-- link_selector = parameters.get('linkSelector', '') or 'a[href]'- url_pattern = parameters.get('urlPattern', '')-- @crawler.router.default_handler- async def handler(ctx: BeautifulSoupCrawlingContext) -> None:- title = ctx.soup.title.string if ctx.soup.title else ''- content = _extract_text(ctx.soup, css_selector, output_format)-- page_data: Dict[str, Any] = {- 'url': ctx.request.url,- 'title': title,- 'content': content,- }- if extract_links:- page_data['links'] = _extract_links(ctx.soup, ctx.request.url)-- pages.append(page_data)-- # Follow links in crawl mode- if mode == 'crawl':- enqueue_kwargs: Dict[str, Any] = {'selector': link_selector}- if url_pattern:- import re- from fnmatch import translate- pattern = re.compile(translate(url_pattern))- enqueue_kwargs['include'] = [pattern]- await ctx.enqueue_links(**enqueue_kwargs)-- await asyncio.wait_for(- crawler.run([url]),- timeout=timeout_secs,- )---async def _run_playwright(- url: str,- pages: List[Dict],- parameters: Dict[str, Any],- proxy_config,- css_selector: str,- extract_links: bool,- output_format: str,- mode: str,- max_pages: int,- max_depth: int,- max_concurrency: int,- timeout_secs: int,-) -> None:- """Run PlaywrightCrawler and collect results into pages list."""- from crawlee import ConcurrencySettings- from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext- from crawlee.storage_clients import MemoryStorageClient-- browser_type = parameters.get('browserType', 'chromium')- wait_for_selector = parameters.get('waitForSelector', '')- wait_timeout = parameters.get('waitTimeout', 30000)- take_screenshot = parameters.get('screenshot', False)-- crawler = PlaywrightCrawler(- browser_type=browser_type,- headless=True,- max_requests_per_crawl=max_pages,- max_crawl_depth=max_depth if mode == 'crawl' else 0,- request_handler_timeout=timedelta(seconds=timeout_secs),- concurrency_settings=ConcurrencySettings(- max_concurrency=max_concurrency,- desired_concurrency=min(max_concurrency, 10),- ),- storage_client=MemoryStorageClient(),- proxy_configuration=proxy_config,- configure_logging=False,- )-- link_selector = parameters.get('linkSelector', '') or 'a[href]'- url_pattern = parameters.get('urlPattern', '')-- @crawler.router.default_handler- async def handler(ctx: PlaywrightCrawlingContext) -> None:- page = ctx.page-- # Wait for specific element if configured- if wait_for_selector:- await page.wait_for_selector(wait_for_selector, timeout=wait_timeout)-- title = await page.title()-- # Get page content for extraction- html = await page.content()- from bs4 import BeautifulSoup- soup = BeautifulSoup(html, 'html.parser')- content = _extract_text(soup, css_selector, output_format)-- page_data: Dict[str, Any] = {- 'url': ctx.request.url,- 'title': title,- 'content': content,- }- if extract_links:- page_data['links'] = _extract_links(soup, ctx.request.url)- if take_screenshot:- screenshot_bytes = await page.screenshot(type='png')- import base64- page_data['screenshot'] = base64.b64encode(screenshot_bytes).decode()-- pages.append(page_data)-- # Follow links in crawl mode- if mode == 'crawl':- enqueue_kwargs: Dict[str, Any] = {'selector': link_selector}- if url_pattern:- import re- from fnmatch import translate- pattern = re.compile(translate(url_pattern))- enqueue_kwargs['include'] = [pattern]- await ctx.enqueue_links(**enqueue_kwargs)-- await asyncio.wait_for(- crawler.run([url]),- timeout=timeout_secs,- )diff --git a/server/services/handlers/tools.py b/server/services/handlers/tools.pyindex 9779d52d..79033e12 100644--- a/server/services/handlers/tools.py+++ b/server/services/handlers/tools.py@@ -186,18 +186,10 @@ async def execute_tool(tool_name: str, tool_args: Dict[str, Any],if node_type == 'taskManager':return await _execute_task_manager(tool_args, config)- # Apify Actor (dual-purpose: workflow node + AI tool)- if node_type == 'apifyActor':- return await _execute_apify_actor(tool_args, config.get('parameters', {}))-# Browser automation (dual-purpose: workflow node + AI tool)if node_type == 'browser':return await _execute_browser(tool_args, config.get('parameters', {}), config)- # Crawlee web scraper (dual-purpose: workflow node + AI tool)- if node_type == 'crawleeScraper':- return await _execute_crawlee_scraper(tool_args, config.get('parameters', {}))-# Proxy nodes (dual-purpose: workflow node + AI tool)# Email nodes (dual-purpose: workflow node + AI tool)if node_type in ('emailSend', 'emailRead'):@@ -1193,78 +1185,6 @@ async def _execute_nearby_places(args: Dict[str, Any],return {"error": f"Nearby places search failed: {str(e)}"}-async def _execute_twitter_send(args: Dict[str, Any],- node_params: Dict[str, Any]) -> Dict[str, Any]:- """Execute Twitter send action via XDK SDK.-- Args:- args: LLM-provided arguments (action, text, tweet_id, reply_to_id)- node_params: Node parameters-- Returns:- Twitter API result- """- from services.handlers.twitter import handle_twitter_send-- parameters = {**node_params, **args}- # Handle reply_to_id alias- if args.get('reply_to_id') and not args.get('tweet_id'):- parameters['tweet_id'] = args['reply_to_id']-- return await handle_twitter_send(- node_id="tool_twitter_send",- node_type="twitterSend",- parameters=parameters,- context={}- )---async def _execute_twitter_search(args: Dict[str, Any],- node_params: Dict[str, Any]) -> Dict[str, Any]:- """Execute Twitter search via XDK SDK.-- Args:- args: LLM-provided arguments (query, max_results)- node_params: Node parameters-- Returns:- Search results- """- from services.handlers.twitter import handle_twitter_search-- parameters = {**node_params, **args}-- return await handle_twitter_search(- node_id="tool_twitter_search",- node_type="twitterSearch",- parameters=parameters,- context={}- )---async def _execute_twitter_user(args: Dict[str, Any],- node_params: Dict[str, Any]) -> Dict[str, Any]:- """Execute Twitter user lookup via XDK SDK.-- Args:- args: LLM-provided arguments (operation, username, user_id, max_results)- node_params: Node parameters-- Returns:- User data- """- from services.handlers.twitter import handle_twitter_user-- parameters = {**node_params, **args}-- return await handle_twitter_user(- node_id="tool_twitter_user",- node_type="twitterUser",- parameters=parameters,- context={}- )--# _execute_google_gmail: deleted in Wave 11.C cleanup. gmail now routes# through the plugin fast-path (nodes/google/gmail.py).@@ -1274,42 +1194,6 @@ async def _execute_twitter_user(args: Dict[str, Any],# (nodes/google/*.py). These functions were defined but unreferenced.-async def _execute_apify_actor(args: Dict[str, Any],- node_params: Dict[str, Any]) -> Dict[str, Any]:- """Execute Apify actor via official SDK.-- Args:- args: LLM-provided arguments (actor_id, input_json, max_results)- node_params: Node parameters-- Returns:- Apify actor run results- """- from services.handlers.apify import handle_apify_actor-- # Parse input_json string to dict if provided- input_json = args.get('input_json', '{}')- if isinstance(input_json, str):- try:- input_json = json.loads(input_json) if input_json.strip() else {}- except json.JSONDecodeError:- input_json = {}-- parameters = {- **node_params,- 'actorId': args.get('actor_id', node_params.get('actorId', '')),- 'actorInput': input_json,- 'maxResults': args.get('max_results', node_params.get('maxResults', 100)),- }-- return await handle_apify_actor(- node_id="tool_apify_actor",- node_type="apifyActor",- parameters=parameters,- context={}- )--async def _execute_browser(args: Dict[str, Any],node_params: Dict[str, Any],config: Dict[str, Any]) -> Dict[str, Any]:@@ -1337,33 +1221,6 @@ async def _execute_browser(args: Dict[str, Any],)-async def _execute_crawlee_scraper(args: Dict[str, Any],- node_params: Dict[str, Any]) -> Dict[str, Any]:- """Execute Crawlee web scraper as an AI tool.-- Maps LLM-provided arguments to node parameters and delegates to handler.- """- from services.handlers.crawlee import handle_crawlee_scraper-- parameters = {- **node_params,- 'url': args.get('url', node_params.get('url', '')),- 'crawlerType': args.get('crawlerType', node_params.get('crawlerType', 'beautifulsoup')),- 'mode': args.get('mode', node_params.get('mode', 'single')),- 'cssSelector': args.get('cssSelector', node_params.get('cssSelector', '')),- 'maxPages': args.get('maxPages', node_params.get('maxPages', 10)),- 'outputFormat': args.get('outputFormat', node_params.get('outputFormat', 'text')),- 'useProxy': args.get('useProxy', node_params.get('useProxy', False)),- }-- return await handle_crawlee_scraper(- node_id="tool_crawlee_scraper",- node_type="crawleeScraper",- parameters=parameters,- context={}- )--async def _execute_proxy_request(args: Dict[str, Any],node_params: Dict[str, Any],config: Dict[str, Any]) -> Dict[str, Any]:diff --git a/server/services/handlers/twitter.py b/server/services/handlers/twitter.pydeleted file mode 100644index 962bdd0d..00000000--- a/server/services/handlers/twitter.py+++ /dev/null@@ -1,652 +0,0 @@-"""Twitter/X node handlers using official XDK SDK."""--import asyncio-import time-from typing import Dict, Any, List-from xdk import Client--from core.logging import get_logger-from services.pricing import get_pricing_service--logger = get_logger(__name__)---async def _track_twitter_usage(- node_id: str,- action: str,- resource_count: int = 1,- workflow_id: str = None,- session_id: str = "default"-) -> Dict[str, float]:- """Track Twitter API usage for cost calculation.-- Args:- node_id: The node executing the Twitter action- action: Action name (tweet, search, followers, etc.)- resource_count: Number of resources fetched (for paginated results)- workflow_id: Optional workflow context- session_id: Session for aggregation-- Returns:- Cost breakdown dict with operation, unit_cost, resource_count, total_cost- """- from core.container import container-- pricing = get_pricing_service()- cost_data = pricing.calculate_api_cost('twitter', action, resource_count)-- # Save to database- db = container.database()- await db.save_api_usage_metric({- 'session_id': session_id,- 'node_id': node_id,- 'workflow_id': workflow_id,- 'service': 'twitter',- 'operation': cost_data.get('operation', action),- 'endpoint': action,- 'resource_count': resource_count,- 'cost': cost_data.get('total_cost', 0.0)- })-- logger.debug(f"[Twitter] Tracked usage: {action} x{resource_count} = ${cost_data.get('total_cost', 0):.6f}")- return cost_data---def _is_auth_error(e: Exception) -> bool:- """Check if exception indicates an expired/invalid token."""- s = str(e)- return '401' in s or '403' in s or 'Unauthorized' in s or 'Forbidden' in s---async def _get_twitter_client() -> Client:- """Get authenticated Twitter client from stored OAuth 2.0 credentials.-- No validation call -- saves rate limits. If the token is expired,- the actual API call will fail and the handler retries via _refresh_and_get_client().- """- from core.container import container- auth_service = container.auth_service()-- tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")- if not tokens or not tokens.get("access_token"):- raise ValueError("Twitter not connected. Please authenticate via Credentials.")-- return Client(access_token=tokens["access_token"])---async def _refresh_and_get_client() -> Client:- """Refresh expired token and return new client."""- from core.container import container- auth_service = container.auth_service()-- tokens = await auth_service.get_oauth_tokens("twitter", customer_id="owner")- refresh_token = tokens.get("refresh_token", "") if tokens else ""-- if not refresh_token:- raise ValueError("Twitter token expired. Please re-authenticate via Credentials.")-- from services.twitter_oauth import TwitterOAuth-- client_id = await auth_service.get_api_key("twitter_client_id") or ""- client_secret = await auth_service.get_api_key("twitter_client_secret")-- oauth = TwitterOAuth(client_id=client_id, client_secret=client_secret, redirect_uri="")- result = await oauth.refresh_access_token(refresh_token)-- if not result.get("success"):- raise ValueError("Twitter token expired and refresh failed. Please re-authenticate.")-- new_access = result["access_token"]- new_refresh = result.get("refresh_token", refresh_token)-- await auth_service.store_oauth_tokens(- provider="twitter",- access_token=new_access,- refresh_token=new_refresh,- email=tokens.get("email", ""),- name=tokens.get("name", ""),- scopes=tokens.get("scopes", ""),- customer_id="owner",- )-- logger.info("Twitter token refreshed successfully")- return Client(access_token=new_access)---async def _get_my_user_id(client: Client) -> str:- """Get the authenticated user's ID."""- response = await asyncio.to_thread(client.users.get_me)- # response.data is a dict with 'id', 'username', 'name' keys- return response.data["id"]---# ----------------------------------------------------------------------------# Sync helper functions for asyncio.to_thread (XDK uses requests internally)-# -----------------------------------------------------------------------------def _sync_create_post(client: Client, payload: dict):- return client.posts.create(body=payload)---def _sync_repost(client: Client, user_id: str, payload: dict):- return client.users.repost_post(user_id, body=payload)---def _sync_like(client: Client, user_id: str, payload: dict):- return client.users.like_post(user_id, body=payload)---def _sync_unlike(client: Client, user_id: str, tweet_id: str):- return client.users.unlike_post(user_id, tweet_id=tweet_id)---def _sync_delete_post(client: Client, tweet_id: str):- return client.posts.delete(tweet_id)---def _sync_search_recent(client: Client, query: str, max_results: int) -> dict:- """Run search_recent synchronously, return first page data + includes.-- Returns dict with 'tweets' (raw tweet data) and 'includes' (expanded- media/user/referenced_tweets objects) so callers can enrich output.- """- for page in client.posts.search_recent(- query=query,- max_results=max_results,- tweet_fields=[- "author_id", "created_at", "entities", "public_metrics",- "possibly_sensitive", "lang", "source",- "conversation_id", "in_reply_to_user_id", "referenced_tweets",- "note_tweet",- ],- expansions=[- "author_id",- "attachments.media_keys",- "referenced_tweets.id",- "referenced_tweets.id.author_id",- ],- media_fields=["url", "preview_image_url", "type", "alt_text", "variants"],- user_fields=["username", "name", "profile_image_url"],- ):- page_data = getattr(page, 'data', []) or []- includes_raw = getattr(page, 'includes', None)- includes_dict = {}- if includes_raw:- if hasattr(includes_raw, 'model_dump'):- try:- includes_dict = includes_raw.model_dump()- except Exception:- includes_dict = {}- elif isinstance(includes_raw, dict):- includes_dict = includes_raw- return {"tweets": page_data, "includes": includes_dict}- return {"tweets": [], "includes": {}}---def _sync_get_me(client: Client, user_fields: list = None):- return client.users.get_me(user_fields=user_fields)---def _sync_get_by_usernames(client: Client, usernames: list, user_fields: list = None):- return client.users.get_by_usernames(usernames=usernames, user_fields=user_fields)---def _sync_get_by_ids(client: Client, ids: list, user_fields: list = None):- return client.users.get_by_ids(ids=ids, user_fields=user_fields)---def _sync_get_followers(client: Client, user_id: str, max_results: int) -> list:- """Get first page of followers synchronously."""- users = []- for page in client.users.get_followers(- user_id, max_results=max_results, user_fields=["created_at"]- ):- page_data = getattr(page, 'data', []) or []- users.extend(page_data)- break- return users---def _sync_get_following(client: Client, user_id: str, max_results: int) -> list:- """Get first page of following synchronously."""- users = []- for page in client.users.get_following(- user_id, max_results=max_results, user_fields=["created_at"]- ):- page_data = getattr(page, 'data', []) or []- users.extend(page_data)- break- return users---async def handle_twitter_send(- node_id: str,- node_type: str,- parameters: Dict[str, Any],- context: Dict[str, Any]-) -> Dict[str, Any]:- """Handle Twitter send actions: tweet, reply, retweet, like, unlike, delete."""- start_time = time.time()- action = parameters.get('action', 'tweet')-- try:- client = await _get_twitter_client()-- # Extract workflow context for tracking- workflow_id = context.get('workflow_id')- session_id = context.get('session_id', 'default')-- try:- return await _do_twitter_send(client, action, parameters, node_id, workflow_id, session_id, start_time)- except Exception as e:- if _is_auth_error(e):- logger.info(f"Twitter auth error, refreshing token: {e}")- client = await _refresh_and_get_client()- return await _do_twitter_send(client, action, parameters, node_id, workflow_id, session_id, start_time)- raise-- except Exception as e:- logger.error(f"Twitter send error: {e}")- return {"success": False, "error": str(e), "execution_time": time.time() - start_time}---async def _do_twitter_send(- client: Client, action: str, parameters: Dict[str, Any],- node_id: str, workflow_id: str, session_id: str, start_time: float-) -> Dict[str, Any]:- """Execute a Twitter send action with the given client."""- match action:- case 'tweet':- text = parameters.get('text', '')- if not text:- raise ValueError("Tweet text is required")- payload = {"text": text[:280]}- result = await asyncio.to_thread(_sync_create_post, client, payload)- await _track_twitter_usage(node_id, 'tweet', 1, workflow_id, session_id)- return _success(_format_response(result), "tweet_sent", start_time)-- case 'reply':- text = parameters.get('text', '')- reply_to = parameters.get('reply_to_id')- if not text or not reply_to:- raise ValueError("Text and reply_to_id are required")- payload = {- "text": text[:280],- "reply": {"in_reply_to_tweet_id": reply_to}- }- result = await asyncio.to_thread(_sync_create_post, client, payload)- await _track_twitter_usage(node_id, 'reply', 1, workflow_id, session_id)- return _success(_format_response(result), "reply_sent", start_time)-- case 'retweet':- tweet_id = parameters.get('tweet_id')- if not tweet_id:- raise ValueError("tweet_id is required")- user_id = await _get_my_user_id(client)- payload = {"tweet_id": tweet_id}- result = await asyncio.to_thread(_sync_repost, client, user_id, payload)- await _track_twitter_usage(node_id, 'retweet', 1, workflow_id, session_id)- return _success(_format_response(result), "retweeted", start_time)-- case 'like':- tweet_id = parameters.get('tweet_id')- if not tweet_id:- raise ValueError("tweet_id is required")- user_id = await _get_my_user_id(client)- payload = {"tweet_id": tweet_id}- result = await asyncio.to_thread(_sync_like, client, user_id, payload)- await _track_twitter_usage(node_id, 'like', 1, workflow_id, session_id)- return _success(_format_response(result), "liked", start_time)-- case 'unlike':- tweet_id = parameters.get('tweet_id')- if not tweet_id:- raise ValueError("tweet_id is required")- user_id = await _get_my_user_id(client)- result = await asyncio.to_thread(_sync_unlike, client, user_id, tweet_id)- await _track_twitter_usage(node_id, 'unlike', 1, workflow_id, session_id)- return _success(_format_response(result), "unliked", start_time)-- case 'delete':- tweet_id = parameters.get('tweet_id')- if not tweet_id:- raise ValueError("tweet_id is required")- result = await asyncio.to_thread(_sync_delete_post, client, tweet_id)- await _track_twitter_usage(node_id, 'delete', 1, workflow_id, session_id)- return _success(_format_response(result), "deleted", start_time)-- case _:- raise ValueError(f"Unknown action: {action}")---async def handle_twitter_search(- node_id: str,- node_type: str,- parameters: Dict[str, Any],- context: Dict[str, Any]-) -> Dict[str, Any]:- """Handle Twitter search operations."""- start_time = time.time()-- try:- client = await _get_twitter_client()- query = parameters.get('query', '')- # X API v2 requires max_results between 10 and 100 for search/recent- max_results = max(10, min(parameters.get('max_results', 10), 100))-- if not query:- raise ValueError("Search query is required")-- # Extract workflow context for tracking- workflow_id = context.get('workflow_id')- session_id = context.get('session_id', 'default')-- try:- search_result = await asyncio.to_thread(_sync_search_recent, client, query, max_results)- except Exception as e:- if _is_auth_error(e):- logger.info(f"Twitter auth error on search, refreshing token: {e}")- client = await _refresh_and_get_client()- search_result = await asyncio.to_thread(_sync_search_recent, client, query, max_results)- else:- raise-- raw_tweets = search_result["tweets"]- includes = search_result["includes"]-- # Build lookup dicts from includes for enriching tweet data- users_by_id = {}- for u in (includes.get("users") or []):- uid = u.get("id") if isinstance(u, dict) else getattr(u, "id", None)- if uid:- users_by_id[str(uid)] = u if isinstance(u, dict) else {- k: v for k, v in u.__dict__.items() if not k.startswith('_')- }-- media_by_key = {}- for m in (includes.get("media") or []):- mk = m.get("media_key") if isinstance(m, dict) else getattr(m, "media_key", None)- if mk:- media_by_key[mk] = m if isinstance(m, dict) else {- k: v for k, v in m.__dict__.items() if not k.startswith('_')- }-- tweets_by_id = {}- for t in (includes.get("tweets") or []):- tid = t.get("id") if isinstance(t, dict) else getattr(t, "id", None)- if tid:- tweets_by_id[str(tid)] = t if isinstance(t, dict) else {- k: v for k, v in t.__dict__.items() if not k.startswith('_')- }-- tweets = [_format_tweet(t, users_by_id, media_by_key, tweets_by_id) for t in raw_tweets]-- # Track: posts_read $0.005 per tweet fetched- if tweets:- await _track_twitter_usage(node_id, 'search', len(tweets), workflow_id, session_id)-- return {- "success": True,- "result": {"tweets": tweets, "count": len(tweets), "query": query},- "execution_time": time.time() - start_time- }-- except Exception as e:- logger.error(f"Twitter search error: {e}")- return {"success": False, "error": str(e), "execution_time": time.time() - start_time}---async def handle_twitter_user(- node_id: str,- node_type: str,- parameters: Dict[str, Any],- context: Dict[str, Any]-) -> Dict[str, Any]:- """Handle Twitter user lookup operations."""- start_time = time.time()- operation = parameters.get('operation', 'me')-- try:- client = await _get_twitter_client()-- # Extract workflow context for tracking- workflow_id = context.get('workflow_id')- session_id = context.get('session_id', 'default')-- try:- return await _do_twitter_user(client, operation, parameters, node_id, workflow_id, session_id, start_time)- except Exception as e:- if _is_auth_error(e):- logger.info(f"Twitter auth error on user op, refreshing token: {e}")- client = await _refresh_and_get_client()- return await _do_twitter_user(client, operation, parameters, node_id, workflow_id, session_id, start_time)- raise-- except Exception as e:- logger.error(f"Twitter user error: {e}")- return {"success": False, "error": str(e), "execution_time": time.time() - start_time}---async def _do_twitter_user(- client: Client, operation: str, parameters: Dict[str, Any],- node_id: str, workflow_id: str, session_id: str, start_time: float-) -> Dict[str, Any]:- """Execute a Twitter user operation with the given client."""- match operation:- case 'me':- result = await asyncio.to_thread(_sync_get_me, client, ["created_at", "description"])- await _track_twitter_usage(node_id, 'me', 1, workflow_id, session_id)- return _success(_format_user_data(result.data), "user", start_time)-- case 'by_username':- username = parameters.get('username')- if not username:- raise ValueError("Username is required")- result = await asyncio.to_thread(- _sync_get_by_usernames, client, [username], ["description", "created_at"]- )- users = getattr(result, 'data', []) or []- if not users:- raise ValueError(f"User @{username} not found")- await _track_twitter_usage(node_id, 'by_username', 1, workflow_id, session_id)- return _success(_format_user_data(users[0]), "user", start_time)-- case 'by_id':- user_id = parameters.get('user_id')- if not user_id:- raise ValueError("User ID is required")- result = await asyncio.to_thread(- _sync_get_by_ids, client, [user_id], ["description", "created_at"]- )- users = getattr(result, 'data', []) or []- if not users:- raise ValueError(f"User ID {user_id} not found")- await _track_twitter_usage(node_id, 'by_id', 1, workflow_id, session_id)- return _success(_format_user_data(users[0]), "user", start_time)-- case 'followers':- user_id = parameters.get('user_id')- if not user_id:- user_id = await _get_my_user_id(client)- # X API requires max_results >= 1 for followers- max_results = max(1, min(parameters.get('max_results', 100), 1000))- raw_users = await asyncio.to_thread(_sync_get_followers, client, user_id, max_results)- users = [_format_user_data(u) for u in raw_users]- if users:- await _track_twitter_usage(node_id, 'followers', len(users), workflow_id, session_id)- return _success({"users": users, "count": len(users)}, "followers", start_time)-- case 'following':- user_id = parameters.get('user_id')- if not user_id:- user_id = await _get_my_user_id(client)- # X API requires max_results >= 1 for following- max_results = max(1, min(parameters.get('max_results', 100), 1000))- raw_users = await asyncio.to_thread(_sync_get_following, client, user_id, max_results)- users = [_format_user_data(u) for u in raw_users]- if users:- await _track_twitter_usage(node_id, 'following', len(users), workflow_id, session_id)- return _success({"users": users, "count": len(users)}, "following", start_time)-- case _:- raise ValueError(f"Unknown operation: {operation}")---def _success(data: Any, action: str, start_time: float) -> Dict[str, Any]:- """Build success response."""- return {- "success": True,- "result": data if isinstance(data, dict) else {"data": data, "action": action},- "execution_time": time.time() - start_time- }---def _format_response(response) -> Dict[str, Any]:- """Format XDK response object to dict."""- if hasattr(response, 'data'):- data = response.data- if isinstance(data, dict):- return data- # Convert object attributes to dict- return {k: v for k, v in data.__dict__.items() if not k.startswith('_')} if hasattr(data, '__dict__') else {"data": str(data)}- return {"response": str(response)}---def _format_tweet(tweet, users_by_id: Dict = None, media_by_key: Dict = None,- tweets_by_id: Dict = None) -> Dict[str, Any]:- """Format tweet object/dict to dict with enriched data from includes.-- Args:- tweet: Raw tweet data (dict or object)- users_by_id: {user_id: user_dict} from includes.users- media_by_key: {media_key: media_dict} from includes.media- tweets_by_id: {tweet_id: tweet_dict} from includes.tweets (referenced)- """- users_by_id = users_by_id or {}- media_by_key = media_by_key or {}- tweets_by_id = tweets_by_id or {}-- def _get(obj, key, default=None):- return obj.get(key, default) if isinstance(obj, dict) else getattr(obj, key, default)-- tweet_id = _get(tweet, 'id')- author_id = _get(tweet, 'author_id')- text = _get(tweet, 'text', '')-- # Extract expanded URLs from entities to replace t.co links- entities = _get(tweet, 'entities') or {}- if isinstance(entities, dict):- urls_list = entities.get('urls', [])- elif hasattr(entities, 'urls'):- urls_list = entities.urls or []- else:- urls_list = []-- expanded_urls = []- display_text = text- for u in urls_list:- if isinstance(u, dict):- short = u.get('url', '')- expanded = u.get('expanded_url', '')- display = u.get('display_url', '')- else:- short = getattr(u, 'url', '')- expanded = getattr(u, 'expanded_url', '')- display = getattr(u, 'display_url', '')- if short and expanded:- expanded_urls.append({"url": short, "expanded_url": expanded, "display_url": display})- display_text = display_text.replace(short, expanded)-- # Note tweet (long-form tweets > 280 chars)- note_tweet = _get(tweet, 'note_tweet')- if note_tweet:- note_text = note_tweet.get('text', '') if isinstance(note_tweet, dict) else getattr(note_tweet, 'text', '')- if note_text:- text = note_text- display_text = note_text-- # Author info from includes- author_info = users_by_id.get(str(author_id)) if author_id else None-- # Media from includes via attachments.media_keys- attachments = _get(tweet, 'attachments') or {}- if isinstance(attachments, dict):- media_keys = attachments.get('media_keys', [])- elif hasattr(attachments, 'media_keys'):- media_keys = attachments.media_keys or []- else:- media_keys = []- media_list = [media_by_key[k] for k in media_keys if k in media_by_key]-- # Referenced tweets (quoted, replied_to)- ref_tweets_raw = _get(tweet, 'referenced_tweets') or []- referenced = []- for ref in ref_tweets_raw:- if isinstance(ref, dict):- ref_type = ref.get('type', '')- ref_id = ref.get('id', '')- else:- ref_type = getattr(ref, 'type', '')- ref_id = getattr(ref, 'id', '')- ref_data = tweets_by_id.get(str(ref_id))- referenced.append({- "type": ref_type,- "id": ref_id,- "text": ref_data.get("text", "") if ref_data else None,- "author_id": ref_data.get("author_id") if ref_data else None,- })-- # Public metrics- metrics = _get(tweet, 'public_metrics') or {}- if not isinstance(metrics, dict) and hasattr(metrics, 'model_dump'):- metrics = metrics.model_dump()- elif not isinstance(metrics, dict):- metrics = {}-- result = {- "id": tweet_id,- "text": text,- "display_text": display_text,- "author_id": author_id,- "created_at": str(_get(tweet, 'created_at', '')),- "lang": _get(tweet, 'lang'),- "source": _get(tweet, 'source'),- "conversation_id": _get(tweet, 'conversation_id'),- "in_reply_to_user_id": _get(tweet, 'in_reply_to_user_id'),- "possibly_sensitive": _get(tweet, 'possibly_sensitive', False),- "public_metrics": metrics,- }-- if author_info:- result["author"] = author_info- if expanded_urls:- result["urls"] = expanded_urls- if media_list:- result["media"] = media_list- if referenced:- result["referenced_tweets"] = referenced-- return result---def _format_user_data(user) -> Dict[str, Any]:- """Format user object/dict to dict."""- if isinstance(user, dict):- return {- "id": user.get("id"),- "username": user.get("username"),- "name": user.get("name"),- "profile_image_url": user.get("profile_image_url"),- "verified": user.get("verified", False),- "description": user.get("description"),- "created_at": str(user.get("created_at", "")),- }- return {- "id": getattr(user, 'id', None),- "username": getattr(user, 'username', None),- "name": getattr(user, 'name', None),- "profile_image_url": getattr(user, 'profile_image_url', None),- "verified": getattr(user, 'verified', False),- "description": getattr(user, 'description', None),- "created_at": str(getattr(user, 'created_at', '')),- }