Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
"""Apify Actor — Wave 11.D.8 inlined.
Runs Apify actors (Instagram, TikTok, Twitter, LinkedIn, Google, Crawler)
via the official apify-client SDK. Quick-input helpers merge into the
raw ``actorInput`` JSON for common actors.
"""
from __future__ import annotations
import json
from datetime import timedelta
from typing import Any, Dict, List, Literal, Optional
from pydantic import BaseModel, ConfigDict, Field
from core.logging import get_logger
from services.plugin import ActionNode, NodeContext, NodeUserError, Operation, TaskQueue
from .._credentials import ApifyCredential
logger = get_logger(__name__)
async def _get_apify_client(*, principal: str | None = None):
"""Return an authenticated Apify client, or None if no token saved."""
from apify_client import ApifyClientAsync # lazy — optional dep
from services.plugin.deps import get_auth_service
auth_service = get_auth_service()
api_token = await auth_service.resolve_api_key("apify", "default", principal=principal)
if not api_token:
return None
return ApifyClientAsync(api_token)
def _build_actor_input(parameters: Dict[str, Any]) -> Dict[str, Any]:
"""Merge actor quick-input helpers into the raw ``actor_input`` JSON.
Apify actors expect camelCase keys in their run input (that's the
SDK contract). The plugin Params are snake_case; we translate here.
"""
actor_id = parameters.get("actor_id", "")
if actor_id == "custom":
actor_id = parameters.get("custom_actor_id", "")
actor_input = parameters.get("actor_input", {})
if isinstance(actor_input, str):
try:
actor_input = json.loads(actor_input) if actor_input.strip() else {}
except json.JSONDecodeError:
actor_input = {}
elif not isinstance(actor_input, dict):
actor_input = {}
if actor_id == "apify/instagram-scraper":
urls = parameters.get("instagram_urls", "")
if urls:
actor_input["directUrls"] = [u.strip() for u in urls.split(",") if u.strip()]
elif actor_id == "clockworks/tiktok-scraper":
profiles = parameters.get("tiktok_profiles", "")
hashtags = parameters.get("tiktok_hashtags", "")
if profiles:
actor_input["profiles"] = [p.strip() for p in profiles.split(",") if p.strip()]
if hashtags:
actor_input["hashtags"] = [h.strip() for h in hashtags.split(",") if h.strip()]
elif actor_id == "apidojo/tweet-scraper":
search_terms = parameters.get("twitter_search_terms", "")
handles = parameters.get("twitter_handles", "")
if search_terms:
actor_input["searchTerms"] = [t.strip() for t in search_terms.split(",") if t.strip()]
if handles:
actor_input["twitterHandles"] = [h.strip() for h in handles.split(",") if h.strip()]
elif actor_id == "apify/google-search-scraper":
query = parameters.get("google_search_query", "")
pages = parameters.get("google_search_pages", 1)
if query:
actor_input["searchQuery"] = query
actor_input["maxPagesPerQuery"] = pages
elif actor_id == "apify/website-content-crawler":
start_urls = parameters.get("crawler_start_urls", "")
max_depth = parameters.get("crawler_max_depth", 2)
max_pages = parameters.get("crawler_max_pages", 50)
if start_urls:
actor_input["startUrls"] = [{"url": u.strip()} for u in start_urls.split(",") if u.strip()]
actor_input["maxCrawlDepth"] = max_depth
actor_input["maxCrawlPages"] = max_pages
return actor_input
async def validate_apify_token(api_token: str) -> Dict[str, Any]:
"""Validate an Apify API token by fetching /users/me.
Used by the websocket ``validate_apify_key`` handler and by the
Credentials modal. Lives on the plugin so there's no handler-file
ghost left behind after Wave 11.D.8.
"""
try:
from apify_client import ApifyClientAsync # lazy — optional dep
client = ApifyClientAsync(api_token)
user_info = await client.user("me").get()
if not user_info:
return {"valid": False, "error": "Could not fetch user info"}
# A token that may not read private info gets the public model,
# which has no email and no plan.
plan = getattr(user_info, "plan", None)
return {
"valid": True,
"username": user_info.username or "",
"email": getattr(user_info, "email", None) or "",
"plan": (plan.id if plan is not None else None) or "free",
}
except Exception as e:
logger.error(f"[Apify] Token validation error: {e}")
msg = str(e)
if "401" in msg or "Unauthorized" in msg:
return {"valid": False, "error": "Invalid API token"}
return {"valid": False, "error": msg}
_ACTOR_PRESETS = [
"apify/instagram-scraper",
"clockworks/tiktok-scraper",
"apidojo/tweet-scraper",
"apify/linkedin-scraper",
"apify/facebook-pages-scraper",
"streamers/youtube-scraper",
"apify/google-search-scraper",
"compass/crawler-google-places",
"apify/website-content-crawler",
"curious_coder/web-scraper",
"custom",
]
class ApifyActorParams(BaseModel):
actor_id: Literal[
"apify/instagram-scraper",
"clockworks/tiktok-scraper",
"apidojo/tweet-scraper",
"apify/linkedin-scraper",
"apify/facebook-pages-scraper",
"streamers/youtube-scraper",
"apify/google-search-scraper",
"compass/crawler-google-places",
"apify/website-content-crawler",
"curious_coder/web-scraper",
"custom",
] = Field(
default="apify/instagram-scraper",
description="Actor preset. Pick 'custom' to enter a specific actor ID.",
)
custom_actor_id: str = Field(
default="",
description="Custom actor ID (e.g. username/actor-name).",
json_schema_extra={"displayOptions": {"show": {"actor_id": ["custom"]}}},
)
actor_input: Any = Field(
default_factory=dict,
description="Raw JSON input passed to the actor (merged with quick-input fields below).",
json_schema_extra={"rows": 6},
)
# Quick-input helpers per actor
instagram_urls: str = Field(
default="",
description="Comma-separated Instagram URLs.",
json_schema_extra={"displayOptions": {"show": {"actor_id": ["apify/instagram-scraper"]}}},
)
tiktok_profiles: str = Field(
default="",
description="Comma-separated TikTok profile handles.",
json_schema_extra={"displayOptions": {"show": {"actor_id": ["clockworks/tiktok-scraper"]}}},
)
tiktok_hashtags: str = Field(
default="",
description="Comma-separated TikTok hashtags.",
json_schema_extra={"displayOptions": {"show": {"actor_id": ["clockworks/tiktok-scraper"]}}},
)
twitter_search_terms: str = Field(
default="",
description="Comma-separated search terms.",
json_schema_extra={"displayOptions": {"show": {"actor_id": ["apidojo/tweet-scraper"]}}},
)
twitter_handles: str = Field(
default="",
description="Comma-separated Twitter handles.",
json_schema_extra={"displayOptions": {"show": {"actor_id": ["apidojo/tweet-scraper"]}}},
)
google_search_query: str = Field(
default="",
description="Google search query.",
json_schema_extra={"displayOptions": {"show": {"actor_id": ["apify/google-search-scraper"]}}},
)
google_search_pages: int = Field(
default=1,
ge=1,
le=100,
description="Max pages per query.",
json_schema_extra={"displayOptions": {"show": {"actor_id": ["apify/google-search-scraper"]}}},
)
crawler_start_urls: str = Field(
default="",
description="Comma-separated start URLs.",
json_schema_extra={"displayOptions": {"show": {"actor_id": ["apify/website-content-crawler"]}}},
)
crawler_max_depth: int = Field(
default=2,
ge=0,
le=20,
description="Max crawl depth.",
json_schema_extra={"displayOptions": {"show": {"actor_id": ["apify/website-content-crawler"]}}},
)
crawler_max_pages: int = Field(
default=50,
ge=1,
le=10000,
description="Max pages to crawl.",
json_schema_extra={"displayOptions": {"show": {"actor_id": ["apify/website-content-crawler"]}}},
)
max_results: int = Field(default=100, ge=1, le=10000)
timeout: int = Field(default=300, ge=1, le=3600)
memory: Literal[128, 256, 512, 1024, 2048, 4096, 8192] = Field(
default=1024,
description="Actor memory in MB.",
)
model_config = ConfigDict(extra="ignore")
class ApifyActorOutput(BaseModel):
run_id: Optional[str] = None
actor_id: Optional[str] = None
status: Optional[str] = None
items: Optional[list] = None
item_count: Optional[int] = None
dataset_id: Optional[str] = None
compute_units: Optional[float] = None
started_at: Optional[str] = None
finished_at: Optional[str] = None
model_config = ConfigDict(extra="allow")
class ApifyActorNode(ActionNode):
type = "apifyActor"
display_name = "Apify Actor"
subtitle = "Web Scraper"
group = ("api", "scraper", "tool")
description = "Run Apify actors for Instagram, TikTok, Twitter, LinkedIn, etc."
component_kind = "square"
tool_name = "apify_actor"
tool_description = "Run Apify actors (pre-built web scrapers) for Instagram, TikTok, Twitter, LinkedIn, Facebook, YouTube, Google Search, Google Maps, and websites. Specify actor_id and actor_input (JSON)."
handles = (
{"name": "input-main", "kind": "input", "position": "left", "label": "Input", "role": "main"},
{"name": "output-main", "kind": "output", "position": "right", "label": "Output", "role": "main"},
)
annotations = {"destructive": False, "readonly": True, "open_world": True}
credentials = (ApifyCredential,)
task_queue = TaskQueue.REST_API
usable_as_tool = True
Params = ApifyActorParams
Output = ApifyActorOutput
@Operation("run")
async def run(self, ctx: NodeContext, params: ApifyActorParams) -> ApifyActorOutput:
client = await _get_apify_client(principal=ctx.raw.get("user_id"))
if not client:
raise NodeUserError(
"Apify API token not configured. Please add your token in Credentials.",
)
actor_id = params.actor_id
if actor_id == "custom":
actor_id = params.custom_actor_id
if not actor_id:
raise NodeUserError("Actor ID is required")
actor_input = _build_actor_input(params.model_dump())
timeout_secs = params.timeout
max_results = params.max_results
memory_mbytes = int(params.memory)
logger.info(
f"[Apify] Running actor {actor_id} " f"timeout={timeout_secs}s memory={memory_mbytes}MB",
)
run = await client.actor(actor_id).call(
run_input=actor_input,
run_timeout=timedelta(seconds=timeout_secs),
memory_mbytes=memory_mbytes,
# The SDK would otherwise stream the actor's own log into ours.
logger=None,
)
if run is None:
raise NodeUserError("Actor run failed - no result returned")
status = run.status
run_id = run.id
dataset_id = run.default_dataset_id
if status == "FAILED":
raise NodeUserError(run.status_message or "Actor run failed")
if status == "TIMED-OUT":
raise NodeUserError("Actor timed out. Try increasing the timeout.")
if status == "ABORTED":
raise NodeUserError("Actor run was aborted")
items: List[Dict[str, Any]] = []
if dataset_id:
listing = await client.dataset(dataset_id).list_items(limit=max_results)
items = listing.items if listing else []
logger.info(f"[Apify] Actor {actor_id} completed: {len(items)} items")
return ApifyActorOutput(
run_id=run_id,
actor_id=actor_id,
status=status,
items=items,
item_count=len(items),
dataset_id=dataset_id,
compute_units=run.usage_total_usd or 0,
started_at=run.started_at.isoformat(),
finished_at=run.finished_at.isoformat() if run.finished_at else "",
)