Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
feat: Add Apify integration for web scraping
trohitga59a4fbe690d
diff --git a/client/src/Dashboard.tsx b/client/src/Dashboard.tsxindex 57417022..882c59b3 100644--- a/client/src/Dashboard.tsx+++ b/client/src/Dashboard.tsx@@ -32,6 +32,7 @@ import { SPECIALIZED_AGENT_TYPES } from './nodeDefinitions/specializedAgentNodesimport { SKILL_NODE_TYPES } from './nodeDefinitions/skillNodes';import { DOCUMENT_NODE_TYPES } from './nodeDefinitions/documentNodes';import { SOCIAL_NODE_TYPES } from './nodeDefinitions/socialNodes';+import { APIFY_NODE_TYPES } from './nodeDefinitions/apifyNodes';import ParameterPanel from './ParameterPanel';import LocationParameterPanel from './components/LocationParameterPanel';import { useAppStore } from './store/useAppStore';@@ -130,6 +131,9 @@ const createNodeTypes = (): Record<string, React.ComponentType<any>> => {} else if (DOCUMENT_NODE_TYPES.includes(type)) {// Document processing nodes use SquareNode componenttypes[type] = SquareNode;+ } else if (APIFY_NODE_TYPES.includes(type)) {+ // Apify web scraping nodes use SquareNode component+ types[type] = SquareNode;} else if (SOCIAL_NODE_TYPES.includes(type)) {// Social nodes use AIAgentNode for multiple output handles supporttypes[type] = AIAgentNode;diff --git a/client/src/assets/icons/apify/apify.svg b/client/src/assets/icons/apify/apify.svgnew file mode 100644index 00000000..fb575161--- /dev/null+++ b/client/src/assets/icons/apify/apify.svg@@ -0,0 +1,5 @@+<svg width="150" height="150" viewBox="0 0 150 150" fill="none" xmlns="http://www.w3.org/2000/svg">+<path d="M86.0214 0H147.727C148.982 0 150 1.01753 150 2.27273V96.5744C150 98.8332 147.062 99.7089 145.826 97.8188L84.1196 3.51714C83.1305 2.00559 84.215 0 86.0214 0Z" fill="#246DFF"/>+<path d="M63.9786 0H2.27273C1.01753 0 0 1.01753 0 2.27273V96.5744C0 98.8332 2.93774 99.7089 4.1745 97.8188L65.8804 3.51714C66.8695 2.00559 65.785 0 63.9786 0Z" fill="#20A34E"/>+<path d="M73.9429 75.5012L3.84485 146.126C2.42137 147.56 3.43724 150 5.45792 150H144.6C146.612 150 147.632 147.578 146.225 146.139L77.1811 75.5135C76.2942 74.6063 74.8365 74.6008 73.9429 75.5012Z" fill="#F86606"/>+</svg>diff --git a/client/src/assets/icons/apify/index.ts b/client/src/assets/icons/apify/index.tsnew file mode 100644index 00000000..9c49b07d--- /dev/null+++ b/client/src/assets/icons/apify/index.ts@@ -0,0 +1,19 @@+// Official Apify icon from n8n-nodes-apify+// Source: https://github.com/apify/n8n-nodes-apify/blob/master/icons/apify.svg++import apifySvg from './apify.svg?raw';++// Convert SVG to data URI for use in node definitions+const svgToDataUri = (svg: string): string => {+ const encoded = encodeURIComponent(svg)+ .replace(/'/g, '%27')+ .replace(/"/g, '%22');+ return `data:image/svg+xml,${encoded}`;+};++export const APIFY_ICON = svgToDataUri(apifySvg);++// Export raw SVG for React component usage+export const ApifyIcons = {+ apify: apifySvg,+};diff --git a/client/src/components/CredentialsModal.tsx b/client/src/components/CredentialsModal.tsxindex 2adf763b..0ffb7990 100644--- a/client/src/components/CredentialsModal.tsx+++ b/client/src/components/CredentialsModal.tsx@@ -24,6 +24,7 @@ import { useWhatsApp } from '../hooks/useWhatsApp';import {OpenAIIcon, ClaudeIcon, GeminiIcon, GroqIcon, OpenRouterIcon, CerebrasIcon,} from './icons/AIProviderIcons';+import { ApifyIcons } from '../assets/icons/apify';// ============================================================================// SERVICE ICONS@@ -56,6 +57,10 @@ const GoogleWorkspaceIcon = () => (<GoogleOutlined style={{ fontSize: 20, color: '#4285F4' }} />);+const ApifyIcon = () => (+ <div dangerouslySetInnerHTML={{ __html: ApifyIcons.apify }} style={{ width: 20, height: 20 }} />+);+// ============================================================================// TYPES & DATA// ============================================================================@@ -112,6 +117,7 @@ const CATEGORIES: Category[] = [label: 'Services',items: [{ id: 'google_maps', name: 'Google Maps', placeholder: 'AIza...', color: '#EA4335', desc: 'Geocoding, Places, Directions', CustomIcon: GoogleMapsIcon, panelType: 'google_maps' },+ { id: 'apify', name: 'Apify', placeholder: 'apify_api_...', color: '#246DFF', desc: 'Web scraping, social media extraction', CustomIcon: ApifyIcon },],},];@@ -128,7 +134,7 @@ interface Props {const CredentialsModal: React.FC<Props> = ({ visible, onClose }) => {const theme = useAppTheme();- const { validateApiKey, saveApiKey, getStoredApiKey, hasStoredKey, removeApiKey, validateGoogleMapsKey, getProviderDefaults, saveProviderDefaults, getProviderUsageSummary, getAPIUsageSummary, getStoredModels, isConnected } = useApiKeys();+const { validateApiKey, saveApiKey, getStoredApiKey, hasStoredKey, removeApiKey, validateGoogleMapsKey, validateApifyKey, getProviderDefaults, saveProviderDefaults, getProviderUsageSummary, getAPIUsageSummary, getStoredModels, isConnected } = useApiKeys();const whatsappStatus = useWhatsAppStatus();const androidStatus = useAndroidStatus();const twitterStatus = useTwitterStatus();@@ -307,9 +313,11 @@ const CredentialsModal: React.FC<Props> = ({ visible, onClose }) => {setLoading(l => ({ ...l, [id]: true }));const result = id === 'google_maps'? await validateGoogleMapsKey(key)- : id === 'android_remote'- ? await saveApiKey(id, key)- : await validateApiKey(id, key);+ : id === 'apify'+ ? await validateApifyKey(key)+ : id === 'android_remote'+ ? await saveApiKey(id, key)+ : await validateApiKey(id, key);setLoading(l => ({ ...l, [id]: false }));if (result.isValid) {diff --git a/client/src/components/parameterPanel/InputSection.tsx b/client/src/components/parameterPanel/InputSection.tsxindex dd2a51ca..75d7cbb4 100644--- a/client/src/components/parameterPanel/InputSection.tsx+++ b/client/src/components/parameterPanel/InputSection.tsx@@ -682,6 +682,18 @@ const InputSection: React.FC<InputSectionProps> = ({ nodeId, visible = true }) =matches: 'array',collection_name: 'string',backend: 'string'+ },+ // Apify web scraping schema+ apify: {+ run_id: 'string',+ actor_id: 'string',+ status: 'string',+ dataset_id: 'string',+ items: 'array',+ item_count: 'number',+ compute_units: 'number',+ started_at: 'string',+ finished_at: 'string'}};@@ -737,6 +749,9 @@ const InputSection: React.FC<InputSectionProps> = ({ nodeId, visible = true }) =const isLocationNode = nodeType.includes('location') && !isAndroidLocation;const isGoogleMaps = nodeType === 'gmaps_create' || nodeType === 'gmaps_locations' || nodeType === 'gmaps_nearby_places';+ // Apify web scraping node detection+ const isApify = nodeType === 'apifyActor';+// Select appropriate schemaif (isAndroidLocation) {outputSchema = sampleSchemas.androidLocation;@@ -764,6 +779,7 @@ const InputSection: React.FC<InputSectionProps> = ({ nodeId, visible = true }) =isTaskTrigger ? sampleSchemas.taskTrigger :isSocialReceive ? sampleSchemas.social :isSocialSend ? sampleSchemas.socialSend :+ isApify ? sampleSchemas.apify :{ data: 'any' };}diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsxindex a8c372b7..c22eac08 100644--- a/client/src/contexts/WebSocketContext.tsx+++ b/client/src/contexts/WebSocketContext.tsx@@ -286,6 +286,9 @@ interface WebSocketContextValue {// Maps OperationsvalidateMapsKey: (apiKey: string) => Promise<{ valid: boolean; message?: string }>;+ // Apify Operations+ validateApifyKey: (apiKey: string) => Promise<{ valid: boolean; message?: string; username?: string }>;+// WhatsApp OperationsgetWhatsAppStatus: () => Promise<{ connected: boolean; deviceId?: string; data?: any }>;getWhatsAppQR: () => Promise<{ connected: boolean; qr?: string; message?: string }>;@@ -1754,6 +1757,26 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi}}, [sendRequest]);+ // =========================================================================+ // Apify Operations+ // =========================================================================++ const validateApifyKeyAsync = useCallback(async (+ apiKey: string+ ): Promise<{ valid: boolean; message?: string; username?: string }> => {+ try {+ const response = await sendRequest<any>('validate_apify_key', { api_key: apiKey });+ return {+ valid: response.valid || false,+ message: response.message,+ username: response.username+ };+ } catch (error) {+ console.error('[WebSocket] Failed to validate Apify key:', error);+ return { valid: false, message: 'Validation failed' };+ }+ }, [sendRequest]);+// =========================================================================// WhatsApp Operations// =========================================================================@@ -2083,6 +2106,9 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi// Maps OperationsvalidateMapsKey: validateMapsKeyAsync,+ // Apify Operations+ validateApifyKey: validateApifyKeyAsync,+// WhatsApp OperationsgetWhatsAppStatus: getWhatsAppStatusAsync,getWhatsAppQR: getWhatsAppQRAsync,diff --git a/client/src/hooks/useApiKeys.ts b/client/src/hooks/useApiKeys.tsindex b16c5195..301c76b6 100644--- a/client/src/hooks/useApiKeys.ts+++ b/client/src/hooks/useApiKeys.ts@@ -87,6 +87,9 @@ export interface UseApiKeysResult {// Validate Google Maps API keyvalidateGoogleMapsKey: (apiKey: string) => Promise<ApiKeyValidationResult>;+ // Validate Apify API key+ validateApifyKey: (apiKey: string) => Promise<ApiKeyValidationResult>;+// Get AI models for a providergetAiModels: (provider: string, apiKey: string) => Promise<string[]>;@@ -113,6 +116,7 @@ export const useApiKeys = (): UseApiKeysResult => {saveApiKey: wsSaveApiKey,deleteApiKey: wsDeleteApiKey,validateMapsKey: wsValidateMapsKey,+ validateApifyKey: wsValidateApifyKey,getAiModels: wsGetAiModels,sendRequest,isConnected@@ -269,6 +273,38 @@ export const useApiKeys = (): UseApiKeysResult => {}}, [wsValidateMapsKey]);+ /**+ * Validate Apify API key+ */+ const validateApifyKey = useCallback(async (+ apiKey: string+ ): Promise<ApiKeyValidationResult> => {+ setIsValidating(true);+ setValidationError(null);++ try {+ const result = await wsValidateApifyKey(apiKey);++ if (!result.valid) {+ setValidationError(result.message || 'Validation failed');+ }++ return {+ isValid: result.valid,+ error: result.message+ };+ } catch (error: any) {+ const errorMsg = error.message || 'Validation failed';+ setValidationError(errorMsg);+ return {+ isValid: false,+ error: errorMsg+ };+ } finally {+ setIsValidating(false);+ }+ }, [wsValidateApifyKey]);+/*** Get available AI models for a provider*/@@ -373,6 +409,7 @@ export const useApiKeys = (): UseApiKeysResult => {getStoredModels,removeApiKey,validateGoogleMapsKey,+ validateApifyKey,getAiModels,getProviderDefaults,saveProviderDefaults,diff --git a/client/src/nodeDefinitions.ts b/client/src/nodeDefinitions.tsindex d9bb9385..031a61ee 100644--- a/client/src/nodeDefinitions.ts+++ b/client/src/nodeDefinitions.ts@@ -18,6 +18,7 @@ import { skillNodes, SKILL_NODE_TYPES } from './nodeDefinitions/skillNodes';import { documentNodes, DOCUMENT_NODE_TYPES } from './nodeDefinitions/documentNodes';import { socialNodes, SOCIAL_NODE_TYPES } from './nodeDefinitions/socialNodes';import { twitterNodes, TWITTER_NODE_TYPES } from './nodeDefinitions/twitterNodes';+import { apifyNodes, APIFY_NODE_TYPES } from './nodeDefinitions/apifyNodes';import { gmailNodes, GMAIL_NODE_TYPES } from './nodeDefinitions/gmailNodes';import { calendarNodes, CALENDAR_NODE_TYPES } from './nodeDefinitions/calendarNodes';import { driveNodes, DRIVE_NODE_TYPES } from './nodeDefinitions/driveNodes';@@ -47,6 +48,7 @@ export const nodeDefinitions: Record<string, INodeTypeDescription> = {...documentNodes,...socialNodes,...twitterNodes,+ ...apifyNodes,...gmailNodes,...calendarNodes,...driveNodes,@@ -128,6 +130,10 @@ export const TWITTER_NODES = [...TWITTER_NODE_TYPES];+export const APIFY_NODES = [+ ...APIFY_NODE_TYPES+];+export const GMAIL_NODES = [...GMAIL_NODE_TYPES];diff --git a/client/src/nodeDefinitions/apifyNodes.ts b/client/src/nodeDefinitions/apifyNodes.tsnew file mode 100644index 00000000..b1ca71b7--- /dev/null+++ b/client/src/nodeDefinitions/apifyNodes.ts@@ -0,0 +1,253 @@+// Apify Node Definitions - Web scraping and social media data extraction+import {+ INodeTypeDescription,+ NodeConnectionType+} from '../types/INodeProperties';+import { APIFY_ICON } from '../assets/icons/apify';++// Re-export icon for external use+export { APIFY_ICON };++// ============================================================================+// APIFY NODES+// ============================================================================++export const apifyNodes: Record<string, INodeTypeDescription> = {+ // Apify Actor - Run any Apify actor (scraper)+ apifyActor: {+ displayName: 'Apify Actor',+ name: 'apifyActor',+ icon: APIFY_ICON,+ group: ['api', 'scraper', 'tool'],+ version: 1,+ subtitle: 'Run Web Scraper',+ description: 'Run web scrapers for social media, websites, and search engines using Apify actors',+ defaults: { name: 'Apify Actor', color: '#FF9012' },+ inputs: [{+ name: 'main',+ displayName: 'Input',+ type: 'main' as NodeConnectionType,+ description: 'Actor input data'+ }],+ outputs: [+ {+ name: 'main',+ displayName: 'Output',+ type: 'main' as NodeConnectionType,+ description: 'Scraped data results'+ },+ {+ name: 'tool',+ displayName: 'Tool',+ type: 'main' as NodeConnectionType,+ description: 'Connect to AI Agent tool handle'+ }+ ],+ properties: [+ // ===== ACTOR SELECTION =====+ {+ displayName: 'Actor',+ name: 'actorId',+ type: 'options',+ options: [+ { name: 'Instagram Scraper', value: 'apify/instagram-scraper' },+ { name: 'TikTok Scraper', value: 'clockworks/tiktok-scraper' },+ { name: 'Twitter/X Scraper', value: 'apidojo/tweet-scraper' },+ { name: 'LinkedIn Profile Scraper', value: 'curious_coder/linkedin-profile-scraper' },+ { name: 'Facebook Posts Scraper', value: 'apify/facebook-posts-scraper' },+ { name: 'YouTube Scraper', value: 'apify/youtube-scraper' },+ { name: 'Google Search Scraper', value: 'apify/google-search-scraper' },+ { name: 'Google Maps Scraper', value: 'apify/google-maps-scraper' },+ { name: 'Website Content Crawler', value: 'apify/website-content-crawler' },+ { name: 'Web Scraper', value: 'apify/web-scraper' },+ { name: 'Custom Actor ID', value: 'custom' }+ ],+ default: 'apify/instagram-scraper',+ required: true,+ description: 'Select a pre-built Apify actor or enter custom actor ID'+ },++ // ===== CUSTOM ACTOR ID (shown when 'custom' selected) =====+ {+ displayName: 'Custom Actor ID',+ name: 'customActorId',+ type: 'string',+ default: '',+ placeholder: 'username/actor-name or actor-id',+ description: 'Enter the actor ID from Apify Store (e.g., apify/cheerio-scraper)',+ displayOptions: {+ show: { actorId: ['custom'] }+ }+ },++ // ===== ACTOR INPUT (JSON) =====+ {+ displayName: 'Actor Input (JSON)',+ name: 'actorInput',+ type: 'json',+ default: '{}',+ typeOptions: { rows: 6 },+ description: 'Input parameters for the actor in JSON format. Each actor has different input schema.'+ },++ // ===== QUICK INPUT HELPERS (Instagram) =====+ {+ displayName: 'Instagram URLs',+ name: 'instagramUrls',+ type: 'string',+ default: '',+ placeholder: 'https://instagram.com/username, https://instagram.com/p/...',+ description: 'Comma-separated Instagram profile or post URLs',+ displayOptions: {+ show: { actorId: ['apify/instagram-scraper'] }+ }+ },++ // ===== QUICK INPUT HELPERS (TikTok) =====+ {+ displayName: 'TikTok Profiles',+ name: 'tiktokProfiles',+ type: 'string',+ default: '',+ placeholder: 'username1, username2',+ description: 'Comma-separated TikTok usernames to scrape',+ displayOptions: {+ show: { actorId: ['clockworks/tiktok-scraper'] }+ }+ },+ {+ displayName: 'TikTok Hashtags',+ name: 'tiktokHashtags',+ type: 'string',+ default: '',+ placeholder: 'trending, fyp',+ description: 'Comma-separated TikTok hashtags to scrape (without #)',+ displayOptions: {+ show: { actorId: ['clockworks/tiktok-scraper'] }+ }+ },++ // ===== QUICK INPUT HELPERS (Twitter) =====+ {+ displayName: 'Twitter Search Terms',+ name: 'twitterSearchTerms',+ type: 'string',+ default: '',+ placeholder: 'AI automation, #tech',+ description: 'Comma-separated search terms or hashtags',+ displayOptions: {+ show: { actorId: ['apidojo/tweet-scraper'] }+ }+ },+ {+ displayName: 'Twitter Handles',+ name: 'twitterHandles',+ type: 'string',+ default: '',+ placeholder: 'elonmusk, openai',+ description: 'Comma-separated Twitter usernames (without @)',+ displayOptions: {+ show: { actorId: ['apidojo/tweet-scraper'] }+ }+ },++ // ===== QUICK INPUT HELPERS (Google Search) =====+ {+ displayName: 'Search Query',+ name: 'googleSearchQuery',+ type: 'string',+ default: '',+ placeholder: 'best AI tools 2026',+ description: 'Google search query',+ displayOptions: {+ show: { actorId: ['apify/google-search-scraper'] }+ }+ },+ {+ displayName: 'Pages to Scrape',+ name: 'googleSearchPages',+ type: 'number',+ default: 1,+ typeOptions: { minValue: 1, maxValue: 10 },+ description: 'Number of search result pages (10 results per page)',+ displayOptions: {+ show: { actorId: ['apify/google-search-scraper'] }+ }+ },++ // ===== QUICK INPUT HELPERS (Web Crawler) =====+ {+ displayName: 'Start URLs',+ name: 'crawlerStartUrls',+ type: 'string',+ default: '',+ placeholder: 'https://example.com, https://docs.example.com',+ description: 'Comma-separated URLs to start crawling from',+ displayOptions: {+ show: { actorId: ['apify/website-content-crawler'] }+ }+ },+ {+ displayName: 'Max Crawl Depth',+ name: 'crawlerMaxDepth',+ type: 'number',+ default: 2,+ typeOptions: { minValue: 0, maxValue: 10 },+ description: 'Maximum link depth to crawl (0 = start URLs only)',+ displayOptions: {+ show: { actorId: ['apify/website-content-crawler'] }+ }+ },+ {+ displayName: 'Max Pages',+ name: 'crawlerMaxPages',+ type: 'number',+ default: 50,+ typeOptions: { minValue: 1, maxValue: 1000 },+ description: 'Maximum number of pages to crawl',+ displayOptions: {+ show: { actorId: ['apify/website-content-crawler'] }+ }+ },++ // ===== EXECUTION OPTIONS =====+ {+ displayName: 'Max Results',+ name: 'maxResults',+ type: 'number',+ default: 100,+ typeOptions: { minValue: 1, maxValue: 10000 },+ description: 'Maximum number of items to return from the dataset'+ },+ {+ displayName: 'Timeout (seconds)',+ name: 'timeout',+ type: 'number',+ default: 300,+ typeOptions: { minValue: 30, maxValue: 3600 },+ description: 'Maximum time to wait for actor completion (30s - 1 hour)'+ },+ {+ displayName: 'Memory (MB)',+ name: 'memory',+ type: 'options',+ options: [+ { name: '128 MB', value: 128 },+ { name: '256 MB', value: 256 },+ { name: '512 MB', value: 512 },+ { name: '1 GB', value: 1024 },+ { name: '2 GB', value: 2048 },+ { name: '4 GB', value: 4096 }+ ],+ default: 1024,+ description: 'Memory allocation for the actor run (affects speed and cost)'+ }+ ]+ }+};++// ============================================================================+// EXPORTS+// ============================================================================++export const APIFY_NODE_TYPES = ['apifyActor'];diff --git a/server/pyproject.toml b/server/pyproject.tomlindex 9425045c..4ad127d8 100644--- a/server/pyproject.toml+++ b/server/pyproject.toml@@ -62,6 +62,9 @@ dependencies = [# Twitter/X API"xdk>=0.1.0",+ # Apify Web Scraping+ "apify-client>=1.8.0",+# Gmail API"google-api-python-client>=2.100.0","google-auth>=2.27.0",diff --git a/server/requirements.txt b/server/requirements.txtindex 4950e19c..cff5e931 100644--- a/server/requirements.txt+++ b/server/requirements.txt@@ -79,6 +79,9 @@ pytz>=2024.1# Twitter/X APIxdk>=0.1.0+# Apify Web Scraping+apify-client>=1.8.0+# Gmail APIgoogle-api-python-client>=2.100.0google-auth>=2.27.0diff --git a/server/routers/websocket.py b/server/routers/websocket.pyindex 04025666..8fe334d8 100644--- a/server/routers/websocket.py+++ b/server/routers/websocket.py@@ -1611,6 +1611,66 @@ async def handle_validate_maps_key(data: Dict[str, Any], websocket: WebSocket) -return {"success": False, "valid": False, "error": str(e)}+# ============================================================================+# Apify Handlers+# ============================================================================++async def handle_validate_apify_key(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:+ """Validate Apify API token and save to database if valid."""+ from services.handlers.apify import validate_apify_token++ broadcaster = get_status_broadcaster()+ auth_service = container.auth_service()++ api_key = data.get("api_key", "").strip()+ session_id = data.get("session_id", "default")++ if not api_key:+ return {"success": False, "valid": False, "error": "api_key required"}++ try:+ result = await validate_apify_token(api_key)++ if result.get("valid"):+ # Save the validated key to database+ await auth_service.store_api_key(+ provider="apify",+ api_key=api_key,+ models=[],+ session_id=session_id+ )+ await broadcaster.update_api_key_status(+ provider="apify",+ valid=True,+ message=f"Apify token validated - user: {result.get('username', 'unknown')}"+ )+ return {+ "success": True,+ "valid": True,+ "message": f"Apify API token is valid",+ "username": result.get("username"),+ "email": result.get("email"),+ "plan": result.get("plan")+ }+ else:+ error_msg = result.get("error", "Invalid API token")+ await broadcaster.update_api_key_status(+ provider="apify",+ valid=False,+ message=error_msg+ )+ return {"success": True, "valid": False, "message": error_msg}++ except Exception as e:+ logger.error("Apify key validation failed", error=str(e))+ await broadcaster.update_api_key_status(+ provider="apify",+ valid=False,+ message=str(e)+ )+ return {"success": False, "valid": False, "error": str(e)}++# ============================================================================# WhatsApp Handlers - Wrappers for routers.whatsapp functions# ============================================================================@@ -2618,6 +2678,9 @@ MESSAGE_HANDLERS: Dict[str, MessageHandler] = {# Maps operations"validate_maps_key": handle_validate_maps_key,+ # Apify operations+ "validate_apify_key": handle_validate_apify_key,+# WhatsApp operations"whatsapp_status": handle_whatsapp_status,"whatsapp_connected_phone": handle_whatsapp_connected_phone,diff --git a/server/services/ai.py b/server/services/ai.pyindex 2386757e..78f4a69f 100644--- a/server/services/ai.py+++ b/server/services/ai.py@@ -3622,6 +3622,34 @@ class AIService:return DelegateToAgentSchema+ # Apify Actor tool schema (dual-purpose: workflow node + AI tool)+ if node_type == 'apifyActor':+ class ApifyActorSchema(BaseModel):+ """Run Apify web scrapers for social media, websites, and search engines.++ Popular actors:+ - apify/instagram-scraper: Scrape Instagram profiles, posts, hashtags+ - clockworks/tiktok-scraper: Scrape TikTok profiles and hashtags+ - apidojo/tweet-scraper: Scrape Twitter/X tweets and users+ - curious_coder/linkedin-profile-scraper: Scrape LinkedIn profiles+ - apify/facebook-posts-scraper: Scrape Facebook posts+ - apify/google-search-scraper: Scrape Google search results+ - apify/website-content-crawler: Crawl any website+ """+ actor_id: str = Field(+ description="Apify actor ID (e.g., 'apify/instagram-scraper', 'clockworks/tiktok-scraper', 'apidojo/tweet-scraper', 'apify/google-search-scraper', 'apify/website-content-crawler')"+ )+ input_json: str = Field(+ default="{}",+ description="Actor input as JSON string. Examples: Instagram: {\"directUrls\": [\"https://instagram.com/username\"]}, TikTok: {\"profiles\": [\"username\"]}, Google: {\"searchQuery\": \"AI tools\"}, Crawler: {\"startUrls\": [{\"url\": \"https://example.com\"}]}"+ )+ max_results: int = Field(+ default=100,+ description="Maximum items to return from the dataset (1-10000)"+ )++ return ApifyActorSchema+# Generic schema for other nodesclass GenericToolSchema(BaseModel):"""Generic schema for tool arguments."""diff --git a/server/services/handlers/apify.py b/server/services/handlers/apify.pynew file mode 100644index 00000000..ce9cc88f--- /dev/null+++ b/server/services/handlers/apify.py@@ -0,0 +1,261 @@+"""Apify node handlers using official apify-client SDK."""++import json+import time+from typing import Dict, Any, Optional, List++from apify_client import ApifyClientAsync++from core.logging import get_logger++logger = get_logger(__name__)+++async def _get_apify_client(context: Dict[str, Any]) -> Optional[ApifyClientAsync]:+ """Get authenticated Apify client from stored credentials."""+ from core.container import container+ auth_service = container.auth_service()+ api_token = await auth_service.get_api_key("apify_api_token")+ if not api_token:+ return None+ return ApifyClientAsync(api_token)+++def _build_actor_input(parameters: Dict[str, Any]) -> Dict[str, Any]:+ """Build actor input from parameters, merging quick helpers with raw JSON input."""+ actor_id = parameters.get('actorId', '')+ if actor_id == 'custom':+ actor_id = parameters.get('customActorId', '')++ # Start with raw JSON input if provided+ actor_input = parameters.get('actorInput', '{}')+ if isinstance(actor_input, str):+ try:+ actor_input = json.loads(actor_input) if actor_input.strip() else {}+ except json.JSONDecodeError:+ actor_input = {}+ elif not isinstance(actor_input, dict):+ actor_input = {}++ # Merge quick helpers based on actor type+ if actor_id == 'apify/instagram-scraper':+ urls = parameters.get('instagramUrls', '')+ if urls:+ actor_input['directUrls'] = [u.strip() for u in urls.split(',') if u.strip()]++ elif actor_id == 'clockworks/tiktok-scraper':+ profiles = parameters.get('tiktokProfiles', '')+ hashtags = parameters.get('tiktokHashtags', '')+ if profiles:+ actor_input['profiles'] = [p.strip() for p in profiles.split(',') if p.strip()]+ if hashtags:+ actor_input['hashtags'] = [h.strip() for h in hashtags.split(',') if h.strip()]++ elif actor_id == 'apidojo/tweet-scraper':+ search_terms = parameters.get('twitterSearchTerms', '')+ handles = parameters.get('twitterHandles', '')+ if search_terms:+ actor_input['searchTerms'] = [t.strip() for t in search_terms.split(',') if t.strip()]+ if handles:+ actor_input['twitterHandles'] = [h.strip() for h in handles.split(',') if h.strip()]++ elif actor_id == 'apify/google-search-scraper':+ query = parameters.get('googleSearchQuery', '')+ pages = parameters.get('googleSearchPages', 1)+ if query:+ actor_input['searchQuery'] = query+ actor_input['maxPagesPerQuery'] = pages++ elif actor_id == 'apify/website-content-crawler':+ start_urls = parameters.get('crawlerStartUrls', '')+ max_depth = parameters.get('crawlerMaxDepth', 2)+ max_pages = parameters.get('crawlerMaxPages', 50)+ if start_urls:+ actor_input['startUrls'] = [{'url': u.strip()} for u in start_urls.split(',') if u.strip()]+ actor_input['maxCrawlDepth'] = max_depth+ actor_input['maxCrawlPages'] = max_pages++ return actor_input+++async def handle_apify_actor(+ node_id: str,+ node_type: str,+ parameters: Dict[str, Any],+ context: Dict[str, Any]+) -> Dict[str, Any]:+ """Run an Apify actor and return results using official SDK.++ Args:+ node_id: Node ID+ node_type: Node type ('apifyActor')+ parameters: Node parameters including actorId, actorInput, maxResults, timeout+ context: Execution context++ Returns:+ Dict with success status, results, and execution metadata+ """+ start_time = time.time()++ try:+ # Get authenticated client+ client = await _get_apify_client(context)+ if not client:+ return {+ "success": False,+ "error": "Apify API token not configured. Please add your token in Credentials.",+ "execution_time": time.time() - start_time+ }++ # Get actor ID+ actor_id = parameters.get("actorId", "")+ if actor_id == "custom":+ actor_id = parameters.get("customActorId", "")+ if not actor_id:+ return {+ "success": False,+ "error": "Actor ID is required",+ "execution_time": time.time() - start_time+ }++ # Build actor input+ actor_input = _build_actor_input(parameters)++ # Get execution options+ timeout_secs = parameters.get("timeout", 300)+ max_results = parameters.get("maxResults", 100)+ memory_mbytes = parameters.get("memory", 1024)++ logger.info(f"[Apify] Running actor {actor_id} with timeout={timeout_secs}s, memory={memory_mbytes}MB")++ # Get actor client and run+ actor_client = client.actor(actor_id)++ # call() runs the actor and waits for it to finish+ # SDK handles polling and retries automatically+ run_info = await actor_client.call(+ run_input=actor_input,+ timeout_secs=timeout_secs,+ memory_mbytes=memory_mbytes+ )++ if run_info is None:+ return {+ "success": False,+ "error": "Actor run failed - no result returned",+ "execution_time": time.time() - start_time+ }++ run_status = run_info.get("status", "UNKNOWN")+ run_id = run_info.get("id", "")+ dataset_id = run_info.get("defaultDatasetId", "")++ # Handle non-success statuses+ if run_status == "FAILED":+ error_msg = run_info.get("errorMessage", "Actor run failed")+ logger.error(f"[Apify] Actor {actor_id} failed: {error_msg}")+ return {+ "success": False,+ "error": error_msg,+ "result": {+ "run_id": run_id,+ "actor_id": actor_id,+ "status": run_status+ },+ "execution_time": time.time() - start_time+ }+ elif run_status == "TIMED-OUT":+ logger.warning(f"[Apify] Actor {actor_id} timed out")+ return {+ "success": False,+ "error": f"Actor timed out. Try increasing the timeout.",+ "result": {+ "run_id": run_id,+ "actor_id": actor_id,+ "status": run_status+ },+ "execution_time": time.time() - start_time+ }+ elif run_status == "ABORTED":+ return {+ "success": False,+ "error": "Actor run was aborted",+ "result": {+ "run_id": run_id,+ "actor_id": actor_id,+ "status": run_status+ },+ "execution_time": time.time() - start_time+ }++ # Get results from dataset+ items: List[Dict[str, Any]] = []+ if dataset_id:+ dataset_client = client.dataset(dataset_id)+ list_result = await dataset_client.list_items(limit=max_results)+ items = list_result.items if list_result else []+ logger.info(f"[Apify] Actor {actor_id} completed: {len(items)} items retrieved")++ # Build result+ result = {+ "run_id": run_id,+ "actor_id": actor_id,+ "status": run_status,+ "items": items,+ "item_count": len(items),+ "dataset_id": dataset_id,+ "compute_units": run_info.get("usageTotalUsd", 0),+ "started_at": run_info.get("startedAt", ""),+ "finished_at": run_info.get("finishedAt", "")+ }++ return {+ "success": True,+ "result": result,+ "execution_time": time.time() - start_time+ }++ except Exception as e:+ logger.error(f"[Apify] Error: {e}")+ error_msg = str(e)+ # Improve error messages for common cases+ if "401" in error_msg or "Unauthorized" in error_msg:+ error_msg = "Invalid Apify API token. Please check your credentials."+ elif "404" in error_msg or "not found" in error_msg.lower():+ error_msg = f"Actor not found. Check the actor ID."+ return {+ "success": False,+ "error": error_msg,+ "execution_time": time.time() - start_time+ }+++async def validate_apify_token(api_token: str) -> Dict[str, Any]:+ """Validate Apify API token by fetching user info.++ Args:+ api_token: Apify API token to validate++ Returns:+ Dict with 'valid' boolean and user info if valid+ """+ try:+ client = ApifyClientAsync(api_token)+ user_client = client.user("me")+ user_info = await user_client.get()++ if user_info:+ return {+ "valid": True,+ "username": user_info.get("username", ""),+ "email": user_info.get("email", ""),+ "plan": user_info.get("plan", {}).get("id", "free") if isinstance(user_info.get("plan"), dict) else "free"+ }+ return {"valid": False, "error": "Could not fetch user info"}++ except Exception as e:+ logger.error(f"[Apify] Token validation error: {e}")+ error_msg = str(e)+ if "401" in error_msg or "Unauthorized" in error_msg:+ return {"valid": False, "error": "Invalid API token"}+ return {"valid": False, "error": str(e)}diff --git a/server/services/handlers/tools.py b/server/services/handlers/tools.pyindex 251f2ebf..512c29e7 100644--- a/server/services/handlers/tools.py+++ b/server/services/handlers/tools.py@@ -209,6 +209,10 @@ async def execute_tool(tool_name: str, tool_args: Dict[str, Any],if node_type == 'taskManager':return await _execute_task_manager(tool_args, config)+ # Apify Actor (dual-purpose: workflow node + AI tool)+ if node_type == 'apifyActor':+ return await _execute_apify_actor(tool_args, config.get('parameters', {}))+# Built-in: Check delegated task results# Auto-injected when parent has delegation toolsif node_type == '_builtin_check_delegated_tasks':@@ -1673,6 +1677,42 @@ async def _execute_contacts_delete(args: Dict[str, Any],)+async def _execute_apify_actor(args: Dict[str, Any],+ node_params: Dict[str, Any]) -> Dict[str, Any]:+ """Execute Apify actor via official SDK.++ Args:+ args: LLM-provided arguments (actor_id, input_json, max_results)+ node_params: Node parameters++ Returns:+ Apify actor run results+ """+ from services.handlers.apify import handle_apify_actor++ # Parse input_json string to dict if provided+ input_json = args.get('input_json', '{}')+ if isinstance(input_json, str):+ try:+ input_json = json.loads(input_json) if input_json.strip() else {}+ except json.JSONDecodeError:+ input_json = {}++ parameters = {+ **node_params,+ 'actorId': args.get('actor_id', node_params.get('actorId', '')),+ 'actorInput': input_json,+ 'maxResults': args.get('max_results', node_params.get('maxResults', 100)),+ }++ return await handle_apify_actor(+ node_id="tool_apify_actor",+ node_type="apifyActor",+ parameters=parameters,+ context={}+ )++async def _execute_generic(args: Dict[str, Any],config: Dict[str, Any]) -> Dict[str, Any]:"""Execute a generic tool (fallback handler).diff --git a/server/services/node_executor.py b/server/services/node_executor.pyindex efc9c6fa..f098552a 100644--- a/server/services/node_executor.py+++ b/server/services/node_executor.py@@ -39,6 +39,7 @@ from services.handlers import (handle_twitter_send, handle_twitter_search, handle_twitter_user,handle_gmail_send, handle_gmail_search, handle_gmail_read,)+from services.handlers.apify import handle_apify_actorfrom services.handlers.calendar import (handle_calendar_create, handle_calendar_list, handle_calendar_update, handle_calendar_delete,)@@ -206,6 +207,8 @@ class NodeExecutor:'taskManager': handle_task_manager,# Team monitoring'teamMonitor': handle_team_monitor,+ # Apify web scraping+ 'apifyActor': handle_apify_actor,# Note: 'console' handled in _dispatch with connected_outputs}diff --git a/server/skills/web_agent/apify-skill/SKILL.md b/server/skills/web_agent/apify-skill/SKILL.mdnew file mode 100644index 00000000..9da2482c--- /dev/null+++ b/server/skills/web_agent/apify-skill/SKILL.md@@ -0,0 +1,237 @@+---+name: apify-skill+description: Run web scrapers and extract data from websites and social media platforms using Apify actors. Supports Instagram, TikTok, Twitter/X, LinkedIn, Facebook, YouTube, Google Search, and general web crawling.+allowed-tools: apify_run_actor+metadata:+ author: machina+ version: "1.0"+ category: web+ icon: "🕷️"+ color: "#FF9012"+---++# Apify Web Scraping Skill++Run pre-built web scrapers (Actors) to extract data from websites and social media platforms.++## How It Works++This skill provides instructions for the **Apify Actor** tool node. Connect the **Apify Actor** node to Zeenie's `input-tools` handle to enable web scraping capabilities.++## apify_run_actor Tool++Run any Apify actor and retrieve scraped results.++### Schema Fields++| Field | Type | Required | Description |+|-------|------|----------|-------------|+| actor_id | string | Yes | Actor ID from Apify Store (e.g., `apify/instagram-scraper`) |+| input_json | string | No | Actor input as JSON string (default: `{}`) |+| max_results | integer | No | Maximum items to return (default: 100) |++### Popular Actors++| Platform | Actor ID | Use Case |+|----------|----------|----------|+| Instagram | `apify/instagram-scraper` | Profiles, posts, hashtags, comments |+| TikTok | `clockworks/tiktok-scraper` | Videos, profiles, trends, hashtags |+| Twitter/X | `apidojo/tweet-scraper` | Tweets, profiles, search results |+| LinkedIn | `curious_coder/linkedin-profile-scraper` | Profiles, companies |+| Facebook | `apify/facebook-posts-scraper` | Posts, pages, groups |+| YouTube | `apify/youtube-scraper` | Videos, channels, comments |+| Google Search | `apify/google-search-scraper` | SERP results, organic listings |+| Google Maps | `apify/google-maps-scraper` | Places, reviews, business info |+| Web Crawler | `apify/website-content-crawler` | Any website content |+| Web Scraper | `apify/web-scraper` | Custom scraping with selectors |++### Response Format++```json+{+ "run_id": "abc123xyz",+ "actor_id": "apify/instagram-scraper",+ "status": "SUCCEEDED",+ "items": [+ { "id": "123", "text": "Post content...", "likes": 500 }+ ],+ "item_count": 50,+ "dataset_id": "dataset-id-here",+ "compute_units": 0.5,+ "started_at": "2026-02-23T10:00:00Z",+ "finished_at": "2026-02-23T10:02:30Z"+}+```++### Examples++**Scrape Instagram profile:**+```json+{+ "actor_id": "apify/instagram-scraper",+ "input_json": "{\"directUrls\": [\"https://instagram.com/natgeo\"], \"resultsLimit\": 50}",+ "max_results": 50+}+```++**Search TikTok hashtag:**+```json+{+ "actor_id": "clockworks/tiktok-scraper",+ "input_json": "{\"hashtags\": [\"trending\", \"fyp\"], \"resultsPerPage\": 30}",+ "max_results": 30+}+```++**Search Twitter/X:**+```json+{+ "actor_id": "apidojo/tweet-scraper",+ "input_json": "{\"searchTerms\": [\"AI automation\"], \"maxItems\": 100}",+ "max_results": 100+}+```++**Google Search:**+```json+{+ "actor_id": "apify/google-search-scraper",+ "input_json": "{\"queries\": \"best AI tools 2026\", \"maxPagesPerQuery\": 3}",+ "max_results": 30+}+```++**Crawl website content:**+```json+{+ "actor_id": "apify/website-content-crawler",+ "input_json": "{\"startUrls\": [{\"url\": \"https://docs.example.com\"}], \"maxCrawlDepth\": 2, \"maxCrawlPages\": 50}",+ "max_results": 50+}+```++**Scrape Google Maps places:**+```json+{+ "actor_id": "apify/google-maps-scraper",+ "input_json": "{\"searchStringsArray\": [\"restaurants in San Francisco\"], \"maxCrawledPlaces\": 20}",+ "max_results": 20+}+```++## Common Input Parameters by Actor++### Instagram Scraper+| Parameter | Type | Description |+|-----------|------|-------------|+| directUrls | array | Instagram URLs to scrape |+| resultsLimit | number | Max results per URL (1-1000) |+| maxComments | number | Comments per post (0-100) |+| searchType | string | Type of search: `hashtag`, `user`, `place` |++### TikTok Scraper+| Parameter | Type | Description |+|-----------|------|-------------|+| profiles | array | TikTok usernames to scrape |+| hashtags | array | Hashtags to scrape (without #) |+| resultsPerPage | number | Results per request |++### Twitter/X Scraper+| Parameter | Type | Description |+|-----------|------|-------------|+| searchTerms | array | Search queries or hashtags |+| twitterHandles | array | Usernames (without @) |+| maxItems | number | Maximum tweets to fetch |++### Google Search Scraper+| Parameter | Type | Description |+|-----------|------|-------------|+| queries | string | Search query |+| maxPagesPerQuery | number | Pages to scrape (10 results/page) |+| languageCode | string | Language filter (e.g., `en`) |+| countryCode | string | Country filter (e.g., `us`) |++### Website Content Crawler+| Parameter | Type | Description |+|-----------|------|-------------|+| startUrls | array | URLs to crawl from (objects with `url` key) |+| maxCrawlDepth | number | Link depth (0 = start URLs only) |+| maxCrawlPages | number | Total pages limit |++## Error Responses++**Actor not found:**+```json+{+ "success": false,+ "error": "Actor 'invalid/actor-id' not found"+}+```++**Timeout:**+```json+{+ "success": false,+ "error": "Actor run timed out after 300 seconds"+}+```++**Run failed:**+```json+{+ "success": false,+ "error": "Actor run failed with status FAILED"+}+```++## Run Status Values++| Status | Meaning |+|--------|---------|+| SUCCEEDED | Run completed successfully |+| FAILED | Run encountered an error |+| ABORTED | Run was manually stopped |+| TIMED-OUT | Run exceeded timeout limit |+| RUNNING | Run still in progress |++## Use Cases++| Use Case | Actor | Description |+|----------|-------|-------------|+| Social listening | tweet-scraper | Monitor brand mentions |+| Competitor research | instagram-scraper | Analyze competitor posts |+| Lead generation | linkedin-profile-scraper | Extract business contacts |+| SEO research | google-search-scraper | Analyze SERP rankings |+| Content aggregation | website-content-crawler | Collect articles from sites |+| Price monitoring | web-scraper | Track product prices |+| Review analysis | google-maps-scraper | Gather customer reviews |++## Guidelines++1. **Actor IDs**: Use format `username/actor-name` from Apify Store+2. **Input JSON**: Must be valid JSON string with actor-specific parameters+3. **Timeouts**: Default 300 seconds, configurable on the node+4. **Max Results**: Limits items returned (not items scraped)+5. **Memory**: Higher memory = faster execution, higher cost++## Pricing Notes++Apify uses compute units (CU) based on memory and duration:+- CU = Memory (GB) x Duration (hours)+- Free tier: $5/month credits+- Check actor pricing on Apify Store before large scrapes++## Setup Requirements++1. Connect the **Apify Actor** node to Zeenie's `input-tools` handle+2. Configure your Apify API token in Credentials Modal+3. Select an actor or enter custom actor ID+4. Provide actor-specific input as JSON++## Security Notes++1. Respect website terms of service+2. Use reasonable scraping rates to avoid IP bans+3. Don't scrape personal data without consent+4. Store scraped data securely+5. Comply with GDPR and data protection laws