Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix: Crawlee memory storage, gitignore update, bump to v0.0.40
trohitg3f0a77fc52af
diff --git a/package.json b/package.jsonindex 884cde14..d1c5eac7 100644--- a/package.json+++ b/package.json@@ -1,6 +1,6 @@{"name": "machinaos",- "version": "0.0.38",+ "version": "0.0.40","description": "Open source workflow automation platform with AI agents, React Flow, and n8n-inspired architecture","type": "module","keywords": [diff --git a/server/.gitignore b/server/.gitignoreindex a1dcba87..fe63e5b7 100644--- a/server/.gitignore+++ b/server/.gitignore@@ -10,4 +10,7 @@ data/*.db*.env.env.local.env.development-.env.production\ No newline at end of file+.env.production++# Crawlee runtime storage (request queues, key-value stores, datasets)+storage/\ No newline at end of filediff --git a/server/services/handlers/crawlee.py b/server/services/handlers/crawlee.pyindex 103b8ecf..6f359d4a 100644--- a/server/services/handlers/crawlee.py+++ b/server/services/handlers/crawlee.py@@ -187,18 +187,20 @@ async def _run_beautifulsoup("""Run BeautifulSoupCrawler and collect results into pages list."""from crawlee import ConcurrencySettingsfrom crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext-- crawler_kwargs: Dict[str, Any] = {- 'max_requests_per_crawl': max_pages,- 'max_crawl_depth': max_depth if mode == 'crawl' else 0,- 'request_handler_timeout': timedelta(seconds=timeout_secs),- 'concurrency_settings': ConcurrencySettings(max_concurrency=max_concurrency),- 'configure_logging': False,- }- if proxy_config:- crawler_kwargs['proxy_configuration'] = proxy_config-- crawler = BeautifulSoupCrawler(**crawler_kwargs)+ from crawlee.storage_clients import MemoryStorageClient++ crawler = BeautifulSoupCrawler(+ max_requests_per_crawl=max_pages,+ max_crawl_depth=max_depth if mode == 'crawl' else 0,+ request_handler_timeout=timedelta(seconds=timeout_secs),+ concurrency_settings=ConcurrencySettings(+ max_concurrency=max_concurrency,+ desired_concurrency=min(max_concurrency, 10),+ ),+ storage_client=MemoryStorageClient(),+ proxy_configuration=proxy_config,+ configure_logging=False,+ )link_selector = parameters.get('linkSelector', '') or 'a[href]'url_pattern = parameters.get('urlPattern', '')@@ -251,25 +253,27 @@ async def _run_playwright("""Run PlaywrightCrawler and collect results into pages list."""from crawlee import ConcurrencySettingsfrom crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext+ from crawlee.storage_clients import MemoryStorageClientbrowser_type = parameters.get('browserType', 'chromium')wait_for_selector = parameters.get('waitForSelector', '')wait_timeout = parameters.get('waitTimeout', 30000)take_screenshot = parameters.get('screenshot', False)- crawler_kwargs: Dict[str, Any] = {- 'max_requests_per_crawl': max_pages,- 'max_crawl_depth': max_depth if mode == 'crawl' else 0,- 'request_handler_timeout': timedelta(seconds=timeout_secs),- 'concurrency_settings': ConcurrencySettings(max_concurrency=max_concurrency),- 'browser_type': browser_type,- 'headless': True,- 'configure_logging': False,- }- if proxy_config:- crawler_kwargs['proxy_configuration'] = proxy_config-- crawler = PlaywrightCrawler(**crawler_kwargs)+ crawler = PlaywrightCrawler(+ browser_type=browser_type,+ headless=True,+ max_requests_per_crawl=max_pages,+ max_crawl_depth=max_depth if mode == 'crawl' else 0,+ request_handler_timeout=timedelta(seconds=timeout_secs),+ concurrency_settings=ConcurrencySettings(+ max_concurrency=max_concurrency,+ desired_concurrency=min(max_concurrency, 10),+ ),+ storage_client=MemoryStorageClient(),+ proxy_configuration=proxy_config,+ configure_logging=False,+ )link_selector = parameters.get('linkSelector', '') or 'a[href]'url_pattern = parameters.get('urlPattern', '')