Skip to content

Self-improving AI Employees for everyone

fix: Crawlee memory storage, gitignore update, bump to v0.0.40

trohitg3f0a77fc52af
diff --git a/package.json b/package.json
index 884cde14..d1c5eac7 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "machinaos",
- "version": "0.0.38",
+ "version": "0.0.40",
"description": "Open source workflow automation platform with AI agents, React Flow, and n8n-inspired architecture",
"type": "module",
"keywords": [
diff --git a/server/.gitignore b/server/.gitignore
index a1dcba87..fe63e5b7 100644
--- a/server/.gitignore
+++ b/server/.gitignore
@@ -10,4 +10,7 @@ data/*.db*
.env
.env.local
.env.development
-.env.production
\ No newline at end of file
+.env.production
+
+# Crawlee runtime storage (request queues, key-value stores, datasets)
+storage/
\ No newline at end of file
diff --git a/server/services/handlers/crawlee.py b/server/services/handlers/crawlee.py
index 103b8ecf..6f359d4a 100644
--- a/server/services/handlers/crawlee.py
+++ b/server/services/handlers/crawlee.py
@@ -187,18 +187,20 @@ async def _run_beautifulsoup(
"""Run BeautifulSoupCrawler and collect results into pages list."""
from crawlee import ConcurrencySettings
from crawlee.crawlers import BeautifulSoupCrawler, BeautifulSoupCrawlingContext
-
- crawler_kwargs: Dict[str, Any] = {
- 'max_requests_per_crawl': max_pages,
- 'max_crawl_depth': max_depth if mode == 'crawl' else 0,
- 'request_handler_timeout': timedelta(seconds=timeout_secs),
- 'concurrency_settings': ConcurrencySettings(max_concurrency=max_concurrency),
- 'configure_logging': False,
- }
- if proxy_config:
- crawler_kwargs['proxy_configuration'] = proxy_config
-
- crawler = BeautifulSoupCrawler(**crawler_kwargs)
+ from crawlee.storage_clients import MemoryStorageClient
+
+ crawler = BeautifulSoupCrawler(
+ max_requests_per_crawl=max_pages,
+ max_crawl_depth=max_depth if mode == 'crawl' else 0,
+ request_handler_timeout=timedelta(seconds=timeout_secs),
+ concurrency_settings=ConcurrencySettings(
+ max_concurrency=max_concurrency,
+ desired_concurrency=min(max_concurrency, 10),
+ ),
+ storage_client=MemoryStorageClient(),
+ proxy_configuration=proxy_config,
+ configure_logging=False,
+ )
link_selector = parameters.get('linkSelector', '') or 'a[href]'
url_pattern = parameters.get('urlPattern', '')
@@ -251,25 +253,27 @@ async def _run_playwright(
"""Run PlaywrightCrawler and collect results into pages list."""
from crawlee import ConcurrencySettings
from crawlee.crawlers import PlaywrightCrawler, PlaywrightCrawlingContext
+ from crawlee.storage_clients import MemoryStorageClient
browser_type = parameters.get('browserType', 'chromium')
wait_for_selector = parameters.get('waitForSelector', '')
wait_timeout = parameters.get('waitTimeout', 30000)
take_screenshot = parameters.get('screenshot', False)
- crawler_kwargs: Dict[str, Any] = {
- 'max_requests_per_crawl': max_pages,
- 'max_crawl_depth': max_depth if mode == 'crawl' else 0,
- 'request_handler_timeout': timedelta(seconds=timeout_secs),
- 'concurrency_settings': ConcurrencySettings(max_concurrency=max_concurrency),
- 'browser_type': browser_type,
- 'headless': True,
- 'configure_logging': False,
- }
- if proxy_config:
- crawler_kwargs['proxy_configuration'] = proxy_config
-
- crawler = PlaywrightCrawler(**crawler_kwargs)
+ crawler = PlaywrightCrawler(
+ browser_type=browser_type,
+ headless=True,
+ max_requests_per_crawl=max_pages,
+ max_crawl_depth=max_depth if mode == 'crawl' else 0,
+ request_handler_timeout=timedelta(seconds=timeout_secs),
+ concurrency_settings=ConcurrencySettings(
+ max_concurrency=max_concurrency,
+ desired_concurrency=min(max_concurrency, 10),
+ ),
+ storage_client=MemoryStorageClient(),
+ proxy_configuration=proxy_config,
+ configure_logging=False,
+ )
link_selector = parameters.get('linkSelector', '') or 'a[href]'
url_pattern = parameters.get('urlPattern', '')