Skip to content

Self-improving AI Employees for everyone

feat: Add Apify integration for web scraping

trohitga59a4fbe690d
diff --git a/client/src/Dashboard.tsx b/client/src/Dashboard.tsx
index 57417022..882c59b3 100644
--- a/client/src/Dashboard.tsx
+++ b/client/src/Dashboard.tsx
@@ -32,6 +32,7 @@ import { SPECIALIZED_AGENT_TYPES } from './nodeDefinitions/specializedAgentNodes
import { SKILL_NODE_TYPES } from './nodeDefinitions/skillNodes';
import { DOCUMENT_NODE_TYPES } from './nodeDefinitions/documentNodes';
import { SOCIAL_NODE_TYPES } from './nodeDefinitions/socialNodes';
+import { APIFY_NODE_TYPES } from './nodeDefinitions/apifyNodes';
import ParameterPanel from './ParameterPanel';
import LocationParameterPanel from './components/LocationParameterPanel';
import { useAppStore } from './store/useAppStore';
@@ -130,6 +131,9 @@ const createNodeTypes = (): Record<string, React.ComponentType<any>> => {
} else if (DOCUMENT_NODE_TYPES.includes(type)) {
// Document processing nodes use SquareNode component
types[type] = SquareNode;
+ } else if (APIFY_NODE_TYPES.includes(type)) {
+ // Apify web scraping nodes use SquareNode component
+ types[type] = SquareNode;
} else if (SOCIAL_NODE_TYPES.includes(type)) {
// Social nodes use AIAgentNode for multiple output handles support
types[type] = AIAgentNode;
diff --git a/client/src/assets/icons/apify/apify.svg b/client/src/assets/icons/apify/apify.svg
new file mode 100644
index 00000000..fb575161
--- /dev/null
+++ b/client/src/assets/icons/apify/apify.svg
@@ -0,0 +1,5 @@
+<svg width="150" height="150" viewBox="0 0 150 150" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M86.0214 0H147.727C148.982 0 150 1.01753 150 2.27273V96.5744C150 98.8332 147.062 99.7089 145.826 97.8188L84.1196 3.51714C83.1305 2.00559 84.215 0 86.0214 0Z" fill="#246DFF"/>
+<path d="M63.9786 0H2.27273C1.01753 0 0 1.01753 0 2.27273V96.5744C0 98.8332 2.93774 99.7089 4.1745 97.8188L65.8804 3.51714C66.8695 2.00559 65.785 0 63.9786 0Z" fill="#20A34E"/>
+<path d="M73.9429 75.5012L3.84485 146.126C2.42137 147.56 3.43724 150 5.45792 150H144.6C146.612 150 147.632 147.578 146.225 146.139L77.1811 75.5135C76.2942 74.6063 74.8365 74.6008 73.9429 75.5012Z" fill="#F86606"/>
+</svg>
diff --git a/client/src/assets/icons/apify/index.ts b/client/src/assets/icons/apify/index.ts
new file mode 100644
index 00000000..9c49b07d
--- /dev/null
+++ b/client/src/assets/icons/apify/index.ts
@@ -0,0 +1,19 @@
+// Official Apify icon from n8n-nodes-apify
+// Source: https://github.com/apify/n8n-nodes-apify/blob/master/icons/apify.svg
+
+import apifySvg from './apify.svg?raw';
+
+// Convert SVG to data URI for use in node definitions
+const svgToDataUri = (svg: string): string => {
+ const encoded = encodeURIComponent(svg)
+ .replace(/'/g, '%27')
+ .replace(/"/g, '%22');
+ return `data:image/svg+xml,${encoded}`;
+};
+
+export const APIFY_ICON = svgToDataUri(apifySvg);
+
+// Export raw SVG for React component usage
+export const ApifyIcons = {
+ apify: apifySvg,
+};
diff --git a/client/src/components/CredentialsModal.tsx b/client/src/components/CredentialsModal.tsx
index 2adf763b..0ffb7990 100644
--- a/client/src/components/CredentialsModal.tsx
+++ b/client/src/components/CredentialsModal.tsx
@@ -24,6 +24,7 @@ import { useWhatsApp } from '../hooks/useWhatsApp';
import {
OpenAIIcon, ClaudeIcon, GeminiIcon, GroqIcon, OpenRouterIcon, CerebrasIcon,
} from './icons/AIProviderIcons';
+import { ApifyIcons } from '../assets/icons/apify';
// ============================================================================
// SERVICE ICONS
@@ -56,6 +57,10 @@ const GoogleWorkspaceIcon = () => (
<GoogleOutlined style={{ fontSize: 20, color: '#4285F4' }} />
);
+const ApifyIcon = () => (
+ <div dangerouslySetInnerHTML={{ __html: ApifyIcons.apify }} style={{ width: 20, height: 20 }} />
+);
+
// ============================================================================
// TYPES & DATA
// ============================================================================
@@ -112,6 +117,7 @@ const CATEGORIES: Category[] = [
label: 'Services',
items: [
{ id: 'google_maps', name: 'Google Maps', placeholder: 'AIza...', color: '#EA4335', desc: 'Geocoding, Places, Directions', CustomIcon: GoogleMapsIcon, panelType: 'google_maps' },
+ { id: 'apify', name: 'Apify', placeholder: 'apify_api_...', color: '#246DFF', desc: 'Web scraping, social media extraction', CustomIcon: ApifyIcon },
],
},
];
@@ -128,7 +134,7 @@ interface Props {
const CredentialsModal: React.FC<Props> = ({ visible, onClose }) => {
const theme = useAppTheme();
- const { validateApiKey, saveApiKey, getStoredApiKey, hasStoredKey, removeApiKey, validateGoogleMapsKey, getProviderDefaults, saveProviderDefaults, getProviderUsageSummary, getAPIUsageSummary, getStoredModels, isConnected } = useApiKeys();
+const { validateApiKey, saveApiKey, getStoredApiKey, hasStoredKey, removeApiKey, validateGoogleMapsKey, validateApifyKey, getProviderDefaults, saveProviderDefaults, getProviderUsageSummary, getAPIUsageSummary, getStoredModels, isConnected } = useApiKeys();
const whatsappStatus = useWhatsAppStatus();
const androidStatus = useAndroidStatus();
const twitterStatus = useTwitterStatus();
@@ -307,9 +313,11 @@ const CredentialsModal: React.FC<Props> = ({ visible, onClose }) => {
setLoading(l => ({ ...l, [id]: true }));
const result = id === 'google_maps'
? await validateGoogleMapsKey(key)
- : id === 'android_remote'
- ? await saveApiKey(id, key)
- : await validateApiKey(id, key);
+ : id === 'apify'
+ ? await validateApifyKey(key)
+ : id === 'android_remote'
+ ? await saveApiKey(id, key)
+ : await validateApiKey(id, key);
setLoading(l => ({ ...l, [id]: false }));
if (result.isValid) {
diff --git a/client/src/components/parameterPanel/InputSection.tsx b/client/src/components/parameterPanel/InputSection.tsx
index dd2a51ca..75d7cbb4 100644
--- a/client/src/components/parameterPanel/InputSection.tsx
+++ b/client/src/components/parameterPanel/InputSection.tsx
@@ -682,6 +682,18 @@ const InputSection: React.FC<InputSectionProps> = ({ nodeId, visible = true }) =
matches: 'array',
collection_name: 'string',
backend: 'string'
+ },
+ // Apify web scraping schema
+ apify: {
+ run_id: 'string',
+ actor_id: 'string',
+ status: 'string',
+ dataset_id: 'string',
+ items: 'array',
+ item_count: 'number',
+ compute_units: 'number',
+ started_at: 'string',
+ finished_at: 'string'
}
};
@@ -737,6 +749,9 @@ const InputSection: React.FC<InputSectionProps> = ({ nodeId, visible = true }) =
const isLocationNode = nodeType.includes('location') && !isAndroidLocation;
const isGoogleMaps = nodeType === 'gmaps_create' || nodeType === 'gmaps_locations' || nodeType === 'gmaps_nearby_places';
+ // Apify web scraping node detection
+ const isApify = nodeType === 'apifyActor';
+
// Select appropriate schema
if (isAndroidLocation) {
outputSchema = sampleSchemas.androidLocation;
@@ -764,6 +779,7 @@ const InputSection: React.FC<InputSectionProps> = ({ nodeId, visible = true }) =
isTaskTrigger ? sampleSchemas.taskTrigger :
isSocialReceive ? sampleSchemas.social :
isSocialSend ? sampleSchemas.socialSend :
+ isApify ? sampleSchemas.apify :
{ data: 'any' };
}
diff --git a/client/src/contexts/WebSocketContext.tsx b/client/src/contexts/WebSocketContext.tsx
index a8c372b7..c22eac08 100644
--- a/client/src/contexts/WebSocketContext.tsx
+++ b/client/src/contexts/WebSocketContext.tsx
@@ -286,6 +286,9 @@ interface WebSocketContextValue {
// Maps Operations
validateMapsKey: (apiKey: string) => Promise<{ valid: boolean; message?: string }>;
+ // Apify Operations
+ validateApifyKey: (apiKey: string) => Promise<{ valid: boolean; message?: string; username?: string }>;
+
// WhatsApp Operations
getWhatsAppStatus: () => Promise<{ connected: boolean; deviceId?: string; data?: any }>;
getWhatsAppQR: () => Promise<{ connected: boolean; qr?: string; message?: string }>;
@@ -1754,6 +1757,26 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
}
}, [sendRequest]);
+ // =========================================================================
+ // Apify Operations
+ // =========================================================================
+
+ const validateApifyKeyAsync = useCallback(async (
+ apiKey: string
+ ): Promise<{ valid: boolean; message?: string; username?: string }> => {
+ try {
+ const response = await sendRequest<any>('validate_apify_key', { api_key: apiKey });
+ return {
+ valid: response.valid || false,
+ message: response.message,
+ username: response.username
+ };
+ } catch (error) {
+ console.error('[WebSocket] Failed to validate Apify key:', error);
+ return { valid: false, message: 'Validation failed' };
+ }
+ }, [sendRequest]);
+
// =========================================================================
// WhatsApp Operations
// =========================================================================
@@ -2083,6 +2106,9 @@ export const WebSocketProvider: React.FC<{ children: React.ReactNode }> = ({ chi
// Maps Operations
validateMapsKey: validateMapsKeyAsync,
+ // Apify Operations
+ validateApifyKey: validateApifyKeyAsync,
+
// WhatsApp Operations
getWhatsAppStatus: getWhatsAppStatusAsync,
getWhatsAppQR: getWhatsAppQRAsync,
diff --git a/client/src/hooks/useApiKeys.ts b/client/src/hooks/useApiKeys.ts
index b16c5195..301c76b6 100644
--- a/client/src/hooks/useApiKeys.ts
+++ b/client/src/hooks/useApiKeys.ts
@@ -87,6 +87,9 @@ export interface UseApiKeysResult {
// Validate Google Maps API key
validateGoogleMapsKey: (apiKey: string) => Promise<ApiKeyValidationResult>;
+ // Validate Apify API key
+ validateApifyKey: (apiKey: string) => Promise<ApiKeyValidationResult>;
+
// Get AI models for a provider
getAiModels: (provider: string, apiKey: string) => Promise<string[]>;
@@ -113,6 +116,7 @@ export const useApiKeys = (): UseApiKeysResult => {
saveApiKey: wsSaveApiKey,
deleteApiKey: wsDeleteApiKey,
validateMapsKey: wsValidateMapsKey,
+ validateApifyKey: wsValidateApifyKey,
getAiModels: wsGetAiModels,
sendRequest,
isConnected
@@ -269,6 +273,38 @@ export const useApiKeys = (): UseApiKeysResult => {
}
}, [wsValidateMapsKey]);
+ /**
+ * Validate Apify API key
+ */
+ const validateApifyKey = useCallback(async (
+ apiKey: string
+ ): Promise<ApiKeyValidationResult> => {
+ setIsValidating(true);
+ setValidationError(null);
+
+ try {
+ const result = await wsValidateApifyKey(apiKey);
+
+ if (!result.valid) {
+ setValidationError(result.message || 'Validation failed');
+ }
+
+ return {
+ isValid: result.valid,
+ error: result.message
+ };
+ } catch (error: any) {
+ const errorMsg = error.message || 'Validation failed';
+ setValidationError(errorMsg);
+ return {
+ isValid: false,
+ error: errorMsg
+ };
+ } finally {
+ setIsValidating(false);
+ }
+ }, [wsValidateApifyKey]);
+
/**
* Get available AI models for a provider
*/
@@ -373,6 +409,7 @@ export const useApiKeys = (): UseApiKeysResult => {
getStoredModels,
removeApiKey,
validateGoogleMapsKey,
+ validateApifyKey,
getAiModels,
getProviderDefaults,
saveProviderDefaults,
diff --git a/client/src/nodeDefinitions.ts b/client/src/nodeDefinitions.ts
index d9bb9385..031a61ee 100644
--- a/client/src/nodeDefinitions.ts
+++ b/client/src/nodeDefinitions.ts
@@ -18,6 +18,7 @@ import { skillNodes, SKILL_NODE_TYPES } from './nodeDefinitions/skillNodes';
import { documentNodes, DOCUMENT_NODE_TYPES } from './nodeDefinitions/documentNodes';
import { socialNodes, SOCIAL_NODE_TYPES } from './nodeDefinitions/socialNodes';
import { twitterNodes, TWITTER_NODE_TYPES } from './nodeDefinitions/twitterNodes';
+import { apifyNodes, APIFY_NODE_TYPES } from './nodeDefinitions/apifyNodes';
import { gmailNodes, GMAIL_NODE_TYPES } from './nodeDefinitions/gmailNodes';
import { calendarNodes, CALENDAR_NODE_TYPES } from './nodeDefinitions/calendarNodes';
import { driveNodes, DRIVE_NODE_TYPES } from './nodeDefinitions/driveNodes';
@@ -47,6 +48,7 @@ export const nodeDefinitions: Record<string, INodeTypeDescription> = {
...documentNodes,
...socialNodes,
...twitterNodes,
+ ...apifyNodes,
...gmailNodes,
...calendarNodes,
...driveNodes,
@@ -128,6 +130,10 @@ export const TWITTER_NODES = [
...TWITTER_NODE_TYPES
];
+export const APIFY_NODES = [
+ ...APIFY_NODE_TYPES
+];
+
export const GMAIL_NODES = [
...GMAIL_NODE_TYPES
];
diff --git a/client/src/nodeDefinitions/apifyNodes.ts b/client/src/nodeDefinitions/apifyNodes.ts
new file mode 100644
index 00000000..b1ca71b7
--- /dev/null
+++ b/client/src/nodeDefinitions/apifyNodes.ts
@@ -0,0 +1,253 @@
+// Apify Node Definitions - Web scraping and social media data extraction
+import {
+ INodeTypeDescription,
+ NodeConnectionType
+} from '../types/INodeProperties';
+import { APIFY_ICON } from '../assets/icons/apify';
+
+// Re-export icon for external use
+export { APIFY_ICON };
+
+// ============================================================================
+// APIFY NODES
+// ============================================================================
+
+export const apifyNodes: Record<string, INodeTypeDescription> = {
+ // Apify Actor - Run any Apify actor (scraper)
+ apifyActor: {
+ displayName: 'Apify Actor',
+ name: 'apifyActor',
+ icon: APIFY_ICON,
+ group: ['api', 'scraper', 'tool'],
+ version: 1,
+ subtitle: 'Run Web Scraper',
+ description: 'Run web scrapers for social media, websites, and search engines using Apify actors',
+ defaults: { name: 'Apify Actor', color: '#FF9012' },
+ inputs: [{
+ name: 'main',
+ displayName: 'Input',
+ type: 'main' as NodeConnectionType,
+ description: 'Actor input data'
+ }],
+ outputs: [
+ {
+ name: 'main',
+ displayName: 'Output',
+ type: 'main' as NodeConnectionType,
+ description: 'Scraped data results'
+ },
+ {
+ name: 'tool',
+ displayName: 'Tool',
+ type: 'main' as NodeConnectionType,
+ description: 'Connect to AI Agent tool handle'
+ }
+ ],
+ properties: [
+ // ===== ACTOR SELECTION =====
+ {
+ displayName: 'Actor',
+ name: 'actorId',
+ type: 'options',
+ options: [
+ { name: 'Instagram Scraper', value: 'apify/instagram-scraper' },
+ { name: 'TikTok Scraper', value: 'clockworks/tiktok-scraper' },
+ { name: 'Twitter/X Scraper', value: 'apidojo/tweet-scraper' },
+ { name: 'LinkedIn Profile Scraper', value: 'curious_coder/linkedin-profile-scraper' },
+ { name: 'Facebook Posts Scraper', value: 'apify/facebook-posts-scraper' },
+ { name: 'YouTube Scraper', value: 'apify/youtube-scraper' },
+ { name: 'Google Search Scraper', value: 'apify/google-search-scraper' },
+ { name: 'Google Maps Scraper', value: 'apify/google-maps-scraper' },
+ { name: 'Website Content Crawler', value: 'apify/website-content-crawler' },
+ { name: 'Web Scraper', value: 'apify/web-scraper' },
+ { name: 'Custom Actor ID', value: 'custom' }
+ ],
+ default: 'apify/instagram-scraper',
+ required: true,
+ description: 'Select a pre-built Apify actor or enter custom actor ID'
+ },
+
+ // ===== CUSTOM ACTOR ID (shown when 'custom' selected) =====
+ {
+ displayName: 'Custom Actor ID',
+ name: 'customActorId',
+ type: 'string',
+ default: '',
+ placeholder: 'username/actor-name or actor-id',
+ description: 'Enter the actor ID from Apify Store (e.g., apify/cheerio-scraper)',
+ displayOptions: {
+ show: { actorId: ['custom'] }
+ }
+ },
+
+ // ===== ACTOR INPUT (JSON) =====
+ {
+ displayName: 'Actor Input (JSON)',
+ name: 'actorInput',
+ type: 'json',
+ default: '{}',
+ typeOptions: { rows: 6 },
+ description: 'Input parameters for the actor in JSON format. Each actor has different input schema.'
+ },
+
+ // ===== QUICK INPUT HELPERS (Instagram) =====
+ {
+ displayName: 'Instagram URLs',
+ name: 'instagramUrls',
+ type: 'string',
+ default: '',
+ placeholder: 'https://instagram.com/username, https://instagram.com/p/...',
+ description: 'Comma-separated Instagram profile or post URLs',
+ displayOptions: {
+ show: { actorId: ['apify/instagram-scraper'] }
+ }
+ },
+
+ // ===== QUICK INPUT HELPERS (TikTok) =====
+ {
+ displayName: 'TikTok Profiles',
+ name: 'tiktokProfiles',
+ type: 'string',
+ default: '',
+ placeholder: 'username1, username2',
+ description: 'Comma-separated TikTok usernames to scrape',
+ displayOptions: {
+ show: { actorId: ['clockworks/tiktok-scraper'] }
+ }
+ },
+ {
+ displayName: 'TikTok Hashtags',
+ name: 'tiktokHashtags',
+ type: 'string',
+ default: '',
+ placeholder: 'trending, fyp',
+ description: 'Comma-separated TikTok hashtags to scrape (without #)',
+ displayOptions: {
+ show: { actorId: ['clockworks/tiktok-scraper'] }
+ }
+ },
+
+ // ===== QUICK INPUT HELPERS (Twitter) =====
+ {
+ displayName: 'Twitter Search Terms',
+ name: 'twitterSearchTerms',
+ type: 'string',
+ default: '',
+ placeholder: 'AI automation, #tech',
+ description: 'Comma-separated search terms or hashtags',
+ displayOptions: {
+ show: { actorId: ['apidojo/tweet-scraper'] }
+ }
+ },
+ {
+ displayName: 'Twitter Handles',
+ name: 'twitterHandles',
+ type: 'string',
+ default: '',
+ placeholder: 'elonmusk, openai',
+ description: 'Comma-separated Twitter usernames (without @)',
+ displayOptions: {
+ show: { actorId: ['apidojo/tweet-scraper'] }
+ }
+ },
+
+ // ===== QUICK INPUT HELPERS (Google Search) =====
+ {
+ displayName: 'Search Query',
+ name: 'googleSearchQuery',
+ type: 'string',
+ default: '',
+ placeholder: 'best AI tools 2026',
+ description: 'Google search query',
+ displayOptions: {
+ show: { actorId: ['apify/google-search-scraper'] }
+ }
+ },
+ {
+ displayName: 'Pages to Scrape',
+ name: 'googleSearchPages',
+ type: 'number',
+ default: 1,
+ typeOptions: { minValue: 1, maxValue: 10 },
+ description: 'Number of search result pages (10 results per page)',
+ displayOptions: {
+ show: { actorId: ['apify/google-search-scraper'] }
+ }
+ },
+
+ // ===== QUICK INPUT HELPERS (Web Crawler) =====
+ {
+ displayName: 'Start URLs',
+ name: 'crawlerStartUrls',
+ type: 'string',
+ default: '',
+ placeholder: 'https://example.com, https://docs.example.com',
+ description: 'Comma-separated URLs to start crawling from',
+ displayOptions: {
+ show: { actorId: ['apify/website-content-crawler'] }
+ }
+ },
+ {
+ displayName: 'Max Crawl Depth',
+ name: 'crawlerMaxDepth',
+ type: 'number',
+ default: 2,
+ typeOptions: { minValue: 0, maxValue: 10 },
+ description: 'Maximum link depth to crawl (0 = start URLs only)',
+ displayOptions: {
+ show: { actorId: ['apify/website-content-crawler'] }
+ }
+ },
+ {
+ displayName: 'Max Pages',
+ name: 'crawlerMaxPages',
+ type: 'number',
+ default: 50,
+ typeOptions: { minValue: 1, maxValue: 1000 },
+ description: 'Maximum number of pages to crawl',
+ displayOptions: {
+ show: { actorId: ['apify/website-content-crawler'] }
+ }
+ },
+
+ // ===== EXECUTION OPTIONS =====
+ {
+ displayName: 'Max Results',
+ name: 'maxResults',
+ type: 'number',
+ default: 100,
+ typeOptions: { minValue: 1, maxValue: 10000 },
+ description: 'Maximum number of items to return from the dataset'
+ },
+ {
+ displayName: 'Timeout (seconds)',
+ name: 'timeout',
+ type: 'number',
+ default: 300,
+ typeOptions: { minValue: 30, maxValue: 3600 },
+ description: 'Maximum time to wait for actor completion (30s - 1 hour)'
+ },
+ {
+ displayName: 'Memory (MB)',
+ name: 'memory',
+ type: 'options',
+ options: [
+ { name: '128 MB', value: 128 },
+ { name: '256 MB', value: 256 },
+ { name: '512 MB', value: 512 },
+ { name: '1 GB', value: 1024 },
+ { name: '2 GB', value: 2048 },
+ { name: '4 GB', value: 4096 }
+ ],
+ default: 1024,
+ description: 'Memory allocation for the actor run (affects speed and cost)'
+ }
+ ]
+ }
+};
+
+// ============================================================================
+// EXPORTS
+// ============================================================================
+
+export const APIFY_NODE_TYPES = ['apifyActor'];
diff --git a/server/pyproject.toml b/server/pyproject.toml
index 9425045c..4ad127d8 100644
--- a/server/pyproject.toml
+++ b/server/pyproject.toml
@@ -62,6 +62,9 @@ dependencies = [
# Twitter/X API
"xdk>=0.1.0",
+ # Apify Web Scraping
+ "apify-client>=1.8.0",
+
# Gmail API
"google-api-python-client>=2.100.0",
"google-auth>=2.27.0",
diff --git a/server/requirements.txt b/server/requirements.txt
index 4950e19c..cff5e931 100644
--- a/server/requirements.txt
+++ b/server/requirements.txt
@@ -79,6 +79,9 @@ pytz>=2024.1
# Twitter/X API
xdk>=0.1.0
+# Apify Web Scraping
+apify-client>=1.8.0
+
# Gmail API
google-api-python-client>=2.100.0
google-auth>=2.27.0
diff --git a/server/routers/websocket.py b/server/routers/websocket.py
index 04025666..8fe334d8 100644
--- a/server/routers/websocket.py
+++ b/server/routers/websocket.py
@@ -1611,6 +1611,66 @@ async def handle_validate_maps_key(data: Dict[str, Any], websocket: WebSocket) -
return {"success": False, "valid": False, "error": str(e)}
+# ============================================================================
+# Apify Handlers
+# ============================================================================
+
+async def handle_validate_apify_key(data: Dict[str, Any], websocket: WebSocket) -> Dict[str, Any]:
+ """Validate Apify API token and save to database if valid."""
+ from services.handlers.apify import validate_apify_token
+
+ broadcaster = get_status_broadcaster()
+ auth_service = container.auth_service()
+
+ api_key = data.get("api_key", "").strip()
+ session_id = data.get("session_id", "default")
+
+ if not api_key:
+ return {"success": False, "valid": False, "error": "api_key required"}
+
+ try:
+ result = await validate_apify_token(api_key)
+
+ if result.get("valid"):
+ # Save the validated key to database
+ await auth_service.store_api_key(
+ provider="apify",
+ api_key=api_key,
+ models=[],
+ session_id=session_id
+ )
+ await broadcaster.update_api_key_status(
+ provider="apify",
+ valid=True,
+ message=f"Apify token validated - user: {result.get('username', 'unknown')}"
+ )
+ return {
+ "success": True,
+ "valid": True,
+ "message": f"Apify API token is valid",
+ "username": result.get("username"),
+ "email": result.get("email"),
+ "plan": result.get("plan")
+ }
+ else:
+ error_msg = result.get("error", "Invalid API token")
+ await broadcaster.update_api_key_status(
+ provider="apify",
+ valid=False,
+ message=error_msg
+ )
+ return {"success": True, "valid": False, "message": error_msg}
+
+ except Exception as e:
+ logger.error("Apify key validation failed", error=str(e))
+ await broadcaster.update_api_key_status(
+ provider="apify",
+ valid=False,
+ message=str(e)
+ )
+ return {"success": False, "valid": False, "error": str(e)}
+
+
# ============================================================================
# WhatsApp Handlers - Wrappers for routers.whatsapp functions
# ============================================================================
@@ -2618,6 +2678,9 @@ MESSAGE_HANDLERS: Dict[str, MessageHandler] = {
# Maps operations
"validate_maps_key": handle_validate_maps_key,
+ # Apify operations
+ "validate_apify_key": handle_validate_apify_key,
+
# WhatsApp operations
"whatsapp_status": handle_whatsapp_status,
"whatsapp_connected_phone": handle_whatsapp_connected_phone,
diff --git a/server/services/ai.py b/server/services/ai.py
index 2386757e..78f4a69f 100644
--- a/server/services/ai.py
+++ b/server/services/ai.py
@@ -3622,6 +3622,34 @@ class AIService:
return DelegateToAgentSchema
+ # Apify Actor tool schema (dual-purpose: workflow node + AI tool)
+ if node_type == 'apifyActor':
+ class ApifyActorSchema(BaseModel):
+ """Run Apify web scrapers for social media, websites, and search engines.
+
+ Popular actors:
+ - apify/instagram-scraper: Scrape Instagram profiles, posts, hashtags
+ - clockworks/tiktok-scraper: Scrape TikTok profiles and hashtags
+ - apidojo/tweet-scraper: Scrape Twitter/X tweets and users
+ - curious_coder/linkedin-profile-scraper: Scrape LinkedIn profiles
+ - apify/facebook-posts-scraper: Scrape Facebook posts
+ - apify/google-search-scraper: Scrape Google search results
+ - apify/website-content-crawler: Crawl any website
+ """
+ actor_id: str = Field(
+ description="Apify actor ID (e.g., 'apify/instagram-scraper', 'clockworks/tiktok-scraper', 'apidojo/tweet-scraper', 'apify/google-search-scraper', 'apify/website-content-crawler')"
+ )
+ input_json: str = Field(
+ default="{}",
+ description="Actor input as JSON string. Examples: Instagram: {\"directUrls\": [\"https://instagram.com/username\"]}, TikTok: {\"profiles\": [\"username\"]}, Google: {\"searchQuery\": \"AI tools\"}, Crawler: {\"startUrls\": [{\"url\": \"https://example.com\"}]}"
+ )
+ max_results: int = Field(
+ default=100,
+ description="Maximum items to return from the dataset (1-10000)"
+ )
+
+ return ApifyActorSchema
+
# Generic schema for other nodes
class GenericToolSchema(BaseModel):
"""Generic schema for tool arguments."""
diff --git a/server/services/handlers/apify.py b/server/services/handlers/apify.py
new file mode 100644
index 00000000..ce9cc88f
--- /dev/null
+++ b/server/services/handlers/apify.py
@@ -0,0 +1,261 @@
+"""Apify node handlers using official apify-client SDK."""
+
+import json
+import time
+from typing import Dict, Any, Optional, List
+
+from apify_client import ApifyClientAsync
+
+from core.logging import get_logger
+
+logger = get_logger(__name__)
+
+
+async def _get_apify_client(context: Dict[str, Any]) -> Optional[ApifyClientAsync]:
+ """Get authenticated Apify client from stored credentials."""
+ from core.container import container
+ auth_service = container.auth_service()
+ api_token = await auth_service.get_api_key("apify_api_token")
+ if not api_token:
+ return None
+ return ApifyClientAsync(api_token)
+
+
+def _build_actor_input(parameters: Dict[str, Any]) -> Dict[str, Any]:
+ """Build actor input from parameters, merging quick helpers with raw JSON input."""
+ actor_id = parameters.get('actorId', '')
+ if actor_id == 'custom':
+ actor_id = parameters.get('customActorId', '')
+
+ # Start with raw JSON input if provided
+ actor_input = parameters.get('actorInput', '{}')
+ if isinstance(actor_input, str):
+ try:
+ actor_input = json.loads(actor_input) if actor_input.strip() else {}
+ except json.JSONDecodeError:
+ actor_input = {}
+ elif not isinstance(actor_input, dict):
+ actor_input = {}
+
+ # Merge quick helpers based on actor type
+ if actor_id == 'apify/instagram-scraper':
+ urls = parameters.get('instagramUrls', '')
+ if urls:
+ actor_input['directUrls'] = [u.strip() for u in urls.split(',') if u.strip()]
+
+ elif actor_id == 'clockworks/tiktok-scraper':
+ profiles = parameters.get('tiktokProfiles', '')
+ hashtags = parameters.get('tiktokHashtags', '')
+ if profiles:
+ actor_input['profiles'] = [p.strip() for p in profiles.split(',') if p.strip()]
+ if hashtags:
+ actor_input['hashtags'] = [h.strip() for h in hashtags.split(',') if h.strip()]
+
+ elif actor_id == 'apidojo/tweet-scraper':
+ search_terms = parameters.get('twitterSearchTerms', '')
+ handles = parameters.get('twitterHandles', '')
+ if search_terms:
+ actor_input['searchTerms'] = [t.strip() for t in search_terms.split(',') if t.strip()]
+ if handles:
+ actor_input['twitterHandles'] = [h.strip() for h in handles.split(',') if h.strip()]
+
+ elif actor_id == 'apify/google-search-scraper':
+ query = parameters.get('googleSearchQuery', '')
+ pages = parameters.get('googleSearchPages', 1)
+ if query:
+ actor_input['searchQuery'] = query
+ actor_input['maxPagesPerQuery'] = pages
+
+ elif actor_id == 'apify/website-content-crawler':
+ start_urls = parameters.get('crawlerStartUrls', '')
+ max_depth = parameters.get('crawlerMaxDepth', 2)
+ max_pages = parameters.get('crawlerMaxPages', 50)
+ if start_urls:
+ actor_input['startUrls'] = [{'url': u.strip()} for u in start_urls.split(',') if u.strip()]
+ actor_input['maxCrawlDepth'] = max_depth
+ actor_input['maxCrawlPages'] = max_pages
+
+ return actor_input
+
+
+async def handle_apify_actor(
+ node_id: str,
+ node_type: str,
+ parameters: Dict[str, Any],
+ context: Dict[str, Any]
+) -> Dict[str, Any]:
+ """Run an Apify actor and return results using official SDK.
+
+ Args:
+ node_id: Node ID
+ node_type: Node type ('apifyActor')
+ parameters: Node parameters including actorId, actorInput, maxResults, timeout
+ context: Execution context
+
+ Returns:
+ Dict with success status, results, and execution metadata
+ """
+ start_time = time.time()
+
+ try:
+ # Get authenticated client
+ client = await _get_apify_client(context)
+ if not client:
+ return {
+ "success": False,
+ "error": "Apify API token not configured. Please add your token in Credentials.",
+ "execution_time": time.time() - start_time
+ }
+
+ # Get actor ID
+ actor_id = parameters.get("actorId", "")
+ if actor_id == "custom":
+ actor_id = parameters.get("customActorId", "")
+ if not actor_id:
+ return {
+ "success": False,
+ "error": "Actor ID is required",
+ "execution_time": time.time() - start_time
+ }
+
+ # Build actor input
+ actor_input = _build_actor_input(parameters)
+
+ # Get execution options
+ timeout_secs = parameters.get("timeout", 300)
+ max_results = parameters.get("maxResults", 100)
+ memory_mbytes = parameters.get("memory", 1024)
+
+ logger.info(f"[Apify] Running actor {actor_id} with timeout={timeout_secs}s, memory={memory_mbytes}MB")
+
+ # Get actor client and run
+ actor_client = client.actor(actor_id)
+
+ # call() runs the actor and waits for it to finish
+ # SDK handles polling and retries automatically
+ run_info = await actor_client.call(
+ run_input=actor_input,
+ timeout_secs=timeout_secs,
+ memory_mbytes=memory_mbytes
+ )
+
+ if run_info is None:
+ return {
+ "success": False,
+ "error": "Actor run failed - no result returned",
+ "execution_time": time.time() - start_time
+ }
+
+ run_status = run_info.get("status", "UNKNOWN")
+ run_id = run_info.get("id", "")
+ dataset_id = run_info.get("defaultDatasetId", "")
+
+ # Handle non-success statuses
+ if run_status == "FAILED":
+ error_msg = run_info.get("errorMessage", "Actor run failed")
+ logger.error(f"[Apify] Actor {actor_id} failed: {error_msg}")
+ return {
+ "success": False,
+ "error": error_msg,
+ "result": {
+ "run_id": run_id,
+ "actor_id": actor_id,
+ "status": run_status
+ },
+ "execution_time": time.time() - start_time
+ }
+ elif run_status == "TIMED-OUT":
+ logger.warning(f"[Apify] Actor {actor_id} timed out")
+ return {
+ "success": False,
+ "error": f"Actor timed out. Try increasing the timeout.",
+ "result": {
+ "run_id": run_id,
+ "actor_id": actor_id,
+ "status": run_status
+ },
+ "execution_time": time.time() - start_time
+ }
+ elif run_status == "ABORTED":
+ return {
+ "success": False,
+ "error": "Actor run was aborted",
+ "result": {
+ "run_id": run_id,
+ "actor_id": actor_id,
+ "status": run_status
+ },
+ "execution_time": time.time() - start_time
+ }
+
+ # Get results from dataset
+ items: List[Dict[str, Any]] = []
+ if dataset_id:
+ dataset_client = client.dataset(dataset_id)
+ list_result = await dataset_client.list_items(limit=max_results)
+ items = list_result.items if list_result else []
+ logger.info(f"[Apify] Actor {actor_id} completed: {len(items)} items retrieved")
+
+ # Build result
+ result = {
+ "run_id": run_id,
+ "actor_id": actor_id,
+ "status": run_status,
+ "items": items,
+ "item_count": len(items),
+ "dataset_id": dataset_id,
+ "compute_units": run_info.get("usageTotalUsd", 0),
+ "started_at": run_info.get("startedAt", ""),
+ "finished_at": run_info.get("finishedAt", "")
+ }
+
+ return {
+ "success": True,
+ "result": result,
+ "execution_time": time.time() - start_time
+ }
+
+ except Exception as e:
+ logger.error(f"[Apify] Error: {e}")
+ error_msg = str(e)
+ # Improve error messages for common cases
+ if "401" in error_msg or "Unauthorized" in error_msg:
+ error_msg = "Invalid Apify API token. Please check your credentials."
+ elif "404" in error_msg or "not found" in error_msg.lower():
+ error_msg = f"Actor not found. Check the actor ID."
+ return {
+ "success": False,
+ "error": error_msg,
+ "execution_time": time.time() - start_time
+ }
+
+
+async def validate_apify_token(api_token: str) -> Dict[str, Any]:
+ """Validate Apify API token by fetching user info.
+
+ Args:
+ api_token: Apify API token to validate
+
+ Returns:
+ Dict with 'valid' boolean and user info if valid
+ """
+ try:
+ client = ApifyClientAsync(api_token)
+ user_client = client.user("me")
+ user_info = await user_client.get()
+
+ if user_info:
+ return {
+ "valid": True,
+ "username": user_info.get("username", ""),
+ "email": user_info.get("email", ""),
+ "plan": user_info.get("plan", {}).get("id", "free") if isinstance(user_info.get("plan"), dict) else "free"
+ }
+ return {"valid": False, "error": "Could not fetch user info"}
+
+ except Exception as e:
+ logger.error(f"[Apify] Token validation error: {e}")
+ error_msg = str(e)
+ if "401" in error_msg or "Unauthorized" in error_msg:
+ return {"valid": False, "error": "Invalid API token"}
+ return {"valid": False, "error": str(e)}
diff --git a/server/services/handlers/tools.py b/server/services/handlers/tools.py
index 251f2ebf..512c29e7 100644
--- a/server/services/handlers/tools.py
+++ b/server/services/handlers/tools.py
@@ -209,6 +209,10 @@ async def execute_tool(tool_name: str, tool_args: Dict[str, Any],
if node_type == 'taskManager':
return await _execute_task_manager(tool_args, config)
+ # Apify Actor (dual-purpose: workflow node + AI tool)
+ if node_type == 'apifyActor':
+ return await _execute_apify_actor(tool_args, config.get('parameters', {}))
+
# Built-in: Check delegated task results
# Auto-injected when parent has delegation tools
if node_type == '_builtin_check_delegated_tasks':
@@ -1673,6 +1677,42 @@ async def _execute_contacts_delete(args: Dict[str, Any],
)
+async def _execute_apify_actor(args: Dict[str, Any],
+ node_params: Dict[str, Any]) -> Dict[str, Any]:
+ """Execute Apify actor via official SDK.
+
+ Args:
+ args: LLM-provided arguments (actor_id, input_json, max_results)
+ node_params: Node parameters
+
+ Returns:
+ Apify actor run results
+ """
+ from services.handlers.apify import handle_apify_actor
+
+ # Parse input_json string to dict if provided
+ input_json = args.get('input_json', '{}')
+ if isinstance(input_json, str):
+ try:
+ input_json = json.loads(input_json) if input_json.strip() else {}
+ except json.JSONDecodeError:
+ input_json = {}
+
+ parameters = {
+ **node_params,
+ 'actorId': args.get('actor_id', node_params.get('actorId', '')),
+ 'actorInput': input_json,
+ 'maxResults': args.get('max_results', node_params.get('maxResults', 100)),
+ }
+
+ return await handle_apify_actor(
+ node_id="tool_apify_actor",
+ node_type="apifyActor",
+ parameters=parameters,
+ context={}
+ )
+
+
async def _execute_generic(args: Dict[str, Any],
config: Dict[str, Any]) -> Dict[str, Any]:
"""Execute a generic tool (fallback handler).
diff --git a/server/services/node_executor.py b/server/services/node_executor.py
index efc9c6fa..f098552a 100644
--- a/server/services/node_executor.py
+++ b/server/services/node_executor.py
@@ -39,6 +39,7 @@ from services.handlers import (
handle_twitter_send, handle_twitter_search, handle_twitter_user,
handle_gmail_send, handle_gmail_search, handle_gmail_read,
)
+from services.handlers.apify import handle_apify_actor
from services.handlers.calendar import (
handle_calendar_create, handle_calendar_list, handle_calendar_update, handle_calendar_delete,
)
@@ -206,6 +207,8 @@ class NodeExecutor:
'taskManager': handle_task_manager,
# Team monitoring
'teamMonitor': handle_team_monitor,
+ # Apify web scraping
+ 'apifyActor': handle_apify_actor,
# Note: 'console' handled in _dispatch with connected_outputs
}
diff --git a/server/skills/web_agent/apify-skill/SKILL.md b/server/skills/web_agent/apify-skill/SKILL.md
new file mode 100644
index 00000000..9da2482c
--- /dev/null
+++ b/server/skills/web_agent/apify-skill/SKILL.md
@@ -0,0 +1,237 @@
+---
+name: apify-skill
+description: Run web scrapers and extract data from websites and social media platforms using Apify actors. Supports Instagram, TikTok, Twitter/X, LinkedIn, Facebook, YouTube, Google Search, and general web crawling.
+allowed-tools: apify_run_actor
+metadata:
+ author: machina
+ version: "1.0"
+ category: web
+ icon: "🕷️"
+ color: "#FF9012"
+---
+
+# Apify Web Scraping Skill
+
+Run pre-built web scrapers (Actors) to extract data from websites and social media platforms.
+
+## How It Works
+
+This skill provides instructions for the **Apify Actor** tool node. Connect the **Apify Actor** node to Zeenie's `input-tools` handle to enable web scraping capabilities.
+
+## apify_run_actor Tool
+
+Run any Apify actor and retrieve scraped results.
+
+### Schema Fields
+
+| Field | Type | Required | Description |
+|-------|------|----------|-------------|
+| actor_id | string | Yes | Actor ID from Apify Store (e.g., `apify/instagram-scraper`) |
+| input_json | string | No | Actor input as JSON string (default: `{}`) |
+| max_results | integer | No | Maximum items to return (default: 100) |
+
+### Popular Actors
+
+| Platform | Actor ID | Use Case |
+|----------|----------|----------|
+| Instagram | `apify/instagram-scraper` | Profiles, posts, hashtags, comments |
+| TikTok | `clockworks/tiktok-scraper` | Videos, profiles, trends, hashtags |
+| Twitter/X | `apidojo/tweet-scraper` | Tweets, profiles, search results |
+| LinkedIn | `curious_coder/linkedin-profile-scraper` | Profiles, companies |
+| Facebook | `apify/facebook-posts-scraper` | Posts, pages, groups |
+| YouTube | `apify/youtube-scraper` | Videos, channels, comments |
+| Google Search | `apify/google-search-scraper` | SERP results, organic listings |
+| Google Maps | `apify/google-maps-scraper` | Places, reviews, business info |
+| Web Crawler | `apify/website-content-crawler` | Any website content |
+| Web Scraper | `apify/web-scraper` | Custom scraping with selectors |
+
+### Response Format
+
+```json
+{
+ "run_id": "abc123xyz",
+ "actor_id": "apify/instagram-scraper",
+ "status": "SUCCEEDED",
+ "items": [
+ { "id": "123", "text": "Post content...", "likes": 500 }
+ ],
+ "item_count": 50,
+ "dataset_id": "dataset-id-here",
+ "compute_units": 0.5,
+ "started_at": "2026-02-23T10:00:00Z",
+ "finished_at": "2026-02-23T10:02:30Z"
+}
+```
+
+### Examples
+
+**Scrape Instagram profile:**
+```json
+{
+ "actor_id": "apify/instagram-scraper",
+ "input_json": "{\"directUrls\": [\"https://instagram.com/natgeo\"], \"resultsLimit\": 50}",
+ "max_results": 50
+}
+```
+
+**Search TikTok hashtag:**
+```json
+{
+ "actor_id": "clockworks/tiktok-scraper",
+ "input_json": "{\"hashtags\": [\"trending\", \"fyp\"], \"resultsPerPage\": 30}",
+ "max_results": 30
+}
+```
+
+**Search Twitter/X:**
+```json
+{
+ "actor_id": "apidojo/tweet-scraper",
+ "input_json": "{\"searchTerms\": [\"AI automation\"], \"maxItems\": 100}",
+ "max_results": 100
+}
+```
+
+**Google Search:**
+```json
+{
+ "actor_id": "apify/google-search-scraper",
+ "input_json": "{\"queries\": \"best AI tools 2026\", \"maxPagesPerQuery\": 3}",
+ "max_results": 30
+}
+```
+
+**Crawl website content:**
+```json
+{
+ "actor_id": "apify/website-content-crawler",
+ "input_json": "{\"startUrls\": [{\"url\": \"https://docs.example.com\"}], \"maxCrawlDepth\": 2, \"maxCrawlPages\": 50}",
+ "max_results": 50
+}
+```
+
+**Scrape Google Maps places:**
+```json
+{
+ "actor_id": "apify/google-maps-scraper",
+ "input_json": "{\"searchStringsArray\": [\"restaurants in San Francisco\"], \"maxCrawledPlaces\": 20}",
+ "max_results": 20
+}
+```
+
+## Common Input Parameters by Actor
+
+### Instagram Scraper
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| directUrls | array | Instagram URLs to scrape |
+| resultsLimit | number | Max results per URL (1-1000) |
+| maxComments | number | Comments per post (0-100) |
+| searchType | string | Type of search: `hashtag`, `user`, `place` |
+
+### TikTok Scraper
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| profiles | array | TikTok usernames to scrape |
+| hashtags | array | Hashtags to scrape (without #) |
+| resultsPerPage | number | Results per request |
+
+### Twitter/X Scraper
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| searchTerms | array | Search queries or hashtags |
+| twitterHandles | array | Usernames (without @) |
+| maxItems | number | Maximum tweets to fetch |
+
+### Google Search Scraper
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| queries | string | Search query |
+| maxPagesPerQuery | number | Pages to scrape (10 results/page) |
+| languageCode | string | Language filter (e.g., `en`) |
+| countryCode | string | Country filter (e.g., `us`) |
+
+### Website Content Crawler
+| Parameter | Type | Description |
+|-----------|------|-------------|
+| startUrls | array | URLs to crawl from (objects with `url` key) |
+| maxCrawlDepth | number | Link depth (0 = start URLs only) |
+| maxCrawlPages | number | Total pages limit |
+
+## Error Responses
+
+**Actor not found:**
+```json
+{
+ "success": false,
+ "error": "Actor 'invalid/actor-id' not found"
+}
+```
+
+**Timeout:**
+```json
+{
+ "success": false,
+ "error": "Actor run timed out after 300 seconds"
+}
+```
+
+**Run failed:**
+```json
+{
+ "success": false,
+ "error": "Actor run failed with status FAILED"
+}
+```
+
+## Run Status Values
+
+| Status | Meaning |
+|--------|---------|
+| SUCCEEDED | Run completed successfully |
+| FAILED | Run encountered an error |
+| ABORTED | Run was manually stopped |
+| TIMED-OUT | Run exceeded timeout limit |
+| RUNNING | Run still in progress |
+
+## Use Cases
+
+| Use Case | Actor | Description |
+|----------|-------|-------------|
+| Social listening | tweet-scraper | Monitor brand mentions |
+| Competitor research | instagram-scraper | Analyze competitor posts |
+| Lead generation | linkedin-profile-scraper | Extract business contacts |
+| SEO research | google-search-scraper | Analyze SERP rankings |
+| Content aggregation | website-content-crawler | Collect articles from sites |
+| Price monitoring | web-scraper | Track product prices |
+| Review analysis | google-maps-scraper | Gather customer reviews |
+
+## Guidelines
+
+1. **Actor IDs**: Use format `username/actor-name` from Apify Store
+2. **Input JSON**: Must be valid JSON string with actor-specific parameters
+3. **Timeouts**: Default 300 seconds, configurable on the node
+4. **Max Results**: Limits items returned (not items scraped)
+5. **Memory**: Higher memory = faster execution, higher cost
+
+## Pricing Notes
+
+Apify uses compute units (CU) based on memory and duration:
+- CU = Memory (GB) x Duration (hours)
+- Free tier: $5/month credits
+- Check actor pricing on Apify Store before large scrapes
+
+## Setup Requirements
+
+1. Connect the **Apify Actor** node to Zeenie's `input-tools` handle
+2. Configure your Apify API token in Credentials Modal
+3. Select an actor or enter custom actor ID
+4. Provide actor-specific input as JSON
+
+## Security Notes
+
+1. Respect website terms of service
+2. Use reasonable scraping rates to avoid IP bans
+3. Don't scrape personal data without consent
+4. Store scraped data securely
+5. Comply with GDPR and data protection laws