AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): implement retry mechanism for SmartDecisionMaker tool call validation (#11015)
Zamil Majdy91dd9364bbda
diff --git a/autogpt_platform/backend/backend/blocks/smart_decision_maker.py b/autogpt_platform/backend/backend/blocks/smart_decision_maker.pyindex 0275898c39..b1432a954f 100644--- a/autogpt_platform/backend/backend/blocks/smart_decision_maker.py+++ b/autogpt_platform/backend/backend/blocks/smart_decision_maker.py@@ -519,34 +519,121 @@ class SmartDecisionMakerBlock(Block):):prompt.append({"role": "user", "content": prefix + input_data.prompt})- response = await llm.llm_call(- credentials=credentials,- llm_model=input_data.model,- prompt=prompt,- max_tokens=input_data.max_tokens,- tools=tool_functions,- ollama_host=input_data.ollama_host,- parallel_tool_calls=input_data.multiple_tool_calls,+ # Use retry decorator for LLM calls with validation+ from backend.util.retry import create_retry_decorator++ # Create retry decorator that excludes ValueError from retry (for non-LLM errors)+ llm_retry = create_retry_decorator(+ max_attempts=input_data.retry,+ exclude_exceptions=(), # Don't exclude ValueError - we want to retry validation failures+ context="SmartDecisionMaker LLM call",)- # Track LLM usage stats- self.merge_stats(- NodeExecutionStats(- input_token_count=response.prompt_tokens,- output_token_count=response.completion_tokens,- llm_call_count=1,+ @llm_retry+ async def call_llm_with_validation():+ response = await llm.llm_call(+ credentials=credentials,+ llm_model=input_data.model,+ prompt=prompt,+ max_tokens=input_data.max_tokens,+ tools=tool_functions,+ ollama_host=input_data.ollama_host,+ parallel_tool_calls=input_data.multiple_tool_calls,)- )++ # Track LLM usage stats+ self.merge_stats(+ NodeExecutionStats(+ input_token_count=response.prompt_tokens,+ output_token_count=response.completion_tokens,+ llm_call_count=1,+ )+ )++ if not response.tool_calls:+ return response, None # No tool calls, return response++ # Validate all tool calls before proceeding+ validation_errors = []+ for tool_call in response.tool_calls:+ tool_name = tool_call.function.name+ tool_args = json.loads(tool_call.function.arguments)++ # Find the tool definition to get the expected arguments+ tool_def = next(+ (+ tool+ for tool in tool_functions+ if tool["function"]["name"] == tool_name+ ),+ None,+ )++ # Get parameters schema from tool definition+ if (+ tool_def+ and "function" in tool_def+ and "parameters" in tool_def["function"]+ ):+ parameters = tool_def["function"]["parameters"]+ expected_args = parameters.get("properties", {})+ required_params = set(parameters.get("required", []))+ else:+ expected_args = {arg: {} for arg in tool_args.keys()}+ required_params = set()++ # Validate tool call arguments+ provided_args = set(tool_args.keys())+ expected_args_set = set(expected_args.keys())++ # Check for unexpected arguments (typos)+ unexpected_args = provided_args - expected_args_set+ # Only check for missing REQUIRED parameters+ missing_required_args = required_params - provided_args++ if unexpected_args or missing_required_args:+ error_msg = f"Tool call '{tool_name}' has parameter errors:"+ if unexpected_args:+ error_msg += f" Unknown parameters: {sorted(unexpected_args)}."+ if missing_required_args:+ error_msg += f" Missing required parameters: {sorted(missing_required_args)}."+ error_msg += f" Expected parameters: {sorted(expected_args_set)}."+ if required_params:+ error_msg += f" Required parameters: {sorted(required_params)}."+ validation_errors.append(error_msg)++ # If validation failed, add feedback and raise for retry+ if validation_errors:+ # Add the failed response to conversation+ prompt.append(response.raw_response)++ # Add error feedback for retry+ error_feedback = (+ "Your tool call had parameter errors. Please fix the following issues and try again:\n"+ + "\n".join(f"- {error}" for error in validation_errors)+ + "\n\nPlease make sure to use the exact parameter names as specified in the function schema."+ )+ prompt.append({"role": "user", "content": error_feedback})++ raise ValueError(+ f"Tool call validation failed: {'; '.join(validation_errors)}"+ )++ return response, validation_errors++ # Call the LLM with retry logic+ response, validation_errors = await call_llm_with_validation()if not response.tool_calls:yield "finished", response.responsereturn+ # If we get here, validation passed - yield tool outputsfor tool_call in response.tool_calls:tool_name = tool_call.function.nametool_args = json.loads(tool_call.function.arguments)- # Find the tool definition to get the expected arguments+ # Get expected arguments (already validated above)tool_def = next((tool@@ -555,39 +642,14 @@ class SmartDecisionMakerBlock(Block):),None,)-- # Get parameters schema from tool definitionif (tool_defand "function" in tool_defand "parameters" in tool_def["function"]):- parameters = tool_def["function"]["parameters"]- expected_args = parameters.get("properties", {})- required_params = set(parameters.get("required", []))+ expected_args = tool_def["function"]["parameters"].get("properties", {})else:expected_args = {arg: {} for arg in tool_args.keys()}- required_params = set()-- # Validate tool call arguments and provide detailed error messages- provided_args = set(tool_args.keys())- expected_args_set = set(expected_args.keys())-- # Check for unexpected arguments (typos)- unexpected_args = provided_args - expected_args_set- # Only check for missing REQUIRED parameters- missing_required_args = required_params - provided_args-- if unexpected_args or missing_required_args:- error_msg = f"Tool call '{tool_name}' has parameter errors:"- if unexpected_args:- error_msg += f" Unknown parameters: {sorted(unexpected_args)}."- if missing_required_args:- error_msg += f" Missing required parameters: {sorted(missing_required_args)}."- error_msg += f" Expected parameters: {sorted(expected_args_set)}."- if required_params:- error_msg += f" Required parameters: {sorted(required_params)}."- raise ValueError(error_msg)# Yield provided arguments, use .get() for optional parametersfor arg_name in expected_args:diff --git a/autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker.py b/autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker.pyindex bd95b97b0a..d8c011149c 100644--- a/autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker.py+++ b/autogpt_platform/backend/backend/blocks/test/test_smart_decision_maker.py@@ -249,3 +249,232 @@ async def test_smart_decision_maker_tracks_llm_stats():# Verify outputsassert "finished" in outputs # Should have finished since no tool callsassert outputs["finished"] == "I need to think about this."+++async def test_smart_decision_maker_parameter_validation():+ """Test that SmartDecisionMakerBlock correctly validates tool call parameters."""+ from unittest.mock import MagicMock, patch++ import backend.blocks.llm as llm_module+ from backend.blocks.smart_decision_maker import SmartDecisionMakerBlock++ block = SmartDecisionMakerBlock()++ # Mock tool functions with specific parameter schema+ mock_tool_functions = [+ {+ "type": "function",+ "function": {+ "name": "search_keywords",+ "description": "Search for keywords with difficulty filtering",+ "parameters": {+ "type": "object",+ "properties": {+ "query": {"type": "string", "description": "Search query"},+ "max_keyword_difficulty": {+ "type": "integer",+ "description": "Maximum keyword difficulty (required)",+ },+ "optional_param": {+ "type": "string",+ "description": "Optional parameter with default",+ "default": "default_value",+ },+ },+ "required": ["query", "max_keyword_difficulty"],+ },+ },+ }+ ]++ # Test case 1: Tool call with TYPO in parameter name (should retry and eventually fail)+ mock_tool_call_with_typo = MagicMock()+ mock_tool_call_with_typo.function.name = "search_keywords"+ mock_tool_call_with_typo.function.arguments = '{"query": "test", "maximum_keyword_difficulty": 50}' # TYPO: maximum instead of max++ mock_response_with_typo = MagicMock()+ mock_response_with_typo.response = None+ mock_response_with_typo.tool_calls = [mock_tool_call_with_typo]+ mock_response_with_typo.prompt_tokens = 50+ mock_response_with_typo.completion_tokens = 25+ mock_response_with_typo.reasoning = None+ mock_response_with_typo.raw_response = {"role": "assistant", "content": None}++ with patch(+ "backend.blocks.llm.llm_call", return_value=mock_response_with_typo+ ) as mock_llm_call, patch.object(+ SmartDecisionMakerBlock,+ "_create_function_signature",+ return_value=mock_tool_functions,+ ):++ input_data = SmartDecisionMakerBlock.Input(+ prompt="Search for keywords",+ model=llm_module.LlmModel.GPT4O,+ credentials=llm_module.TEST_CREDENTIALS_INPUT, # type: ignore+ retry=2, # Set retry to 2 for testing+ )++ # Should raise ValueError after retries due to typo'd parameter name+ with pytest.raises(ValueError) as exc_info:+ outputs = {}+ async for output_name, output_data in block.run(+ input_data,+ credentials=llm_module.TEST_CREDENTIALS,+ graph_id="test-graph-id",+ node_id="test-node-id",+ graph_exec_id="test-exec-id",+ node_exec_id="test-node-exec-id",+ user_id="test-user-id",+ ):+ outputs[output_name] = output_data++ # Verify error message contains details about the typo+ error_msg = str(exc_info.value)+ assert "Tool call validation failed" in error_msg+ assert "Unknown parameters: ['maximum_keyword_difficulty']" in error_msg++ # Verify that LLM was called the expected number of times (retries)+ assert mock_llm_call.call_count == 2 # Should retry based on input_data.retry++ # Test case 2: Tool call missing REQUIRED parameter (should raise ValueError)+ mock_tool_call_missing_required = MagicMock()+ mock_tool_call_missing_required.function.name = "search_keywords"+ mock_tool_call_missing_required.function.arguments = (+ '{"query": "test"}' # Missing required max_keyword_difficulty+ )++ mock_response_missing_required = MagicMock()+ mock_response_missing_required.response = None+ mock_response_missing_required.tool_calls = [mock_tool_call_missing_required]+ mock_response_missing_required.prompt_tokens = 50+ mock_response_missing_required.completion_tokens = 25+ mock_response_missing_required.reasoning = None+ mock_response_missing_required.raw_response = {"role": "assistant", "content": None}++ with patch(+ "backend.blocks.llm.llm_call", return_value=mock_response_missing_required+ ), patch.object(+ SmartDecisionMakerBlock,+ "_create_function_signature",+ return_value=mock_tool_functions,+ ):++ input_data = SmartDecisionMakerBlock.Input(+ prompt="Search for keywords",+ model=llm_module.LlmModel.GPT4O,+ credentials=llm_module.TEST_CREDENTIALS_INPUT, # type: ignore+ )++ # Should raise ValueError due to missing required parameter+ with pytest.raises(ValueError) as exc_info:+ outputs = {}+ async for output_name, output_data in block.run(+ input_data,+ credentials=llm_module.TEST_CREDENTIALS,+ graph_id="test-graph-id",+ node_id="test-node-id",+ graph_exec_id="test-exec-id",+ node_exec_id="test-node-exec-id",+ user_id="test-user-id",+ ):+ outputs[output_name] = output_data++ error_msg = str(exc_info.value)+ assert "Tool call 'search_keywords' has parameter errors" in error_msg+ assert "Missing required parameters: ['max_keyword_difficulty']" in error_msg++ # Test case 3: Valid tool call with OPTIONAL parameter missing (should succeed)+ mock_tool_call_valid = MagicMock()+ mock_tool_call_valid.function.name = "search_keywords"+ mock_tool_call_valid.function.arguments = '{"query": "test", "max_keyword_difficulty": 50}' # optional_param missing, but that's OK++ mock_response_valid = MagicMock()+ mock_response_valid.response = None+ mock_response_valid.tool_calls = [mock_tool_call_valid]+ mock_response_valid.prompt_tokens = 50+ mock_response_valid.completion_tokens = 25+ mock_response_valid.reasoning = None+ mock_response_valid.raw_response = {"role": "assistant", "content": None}++ with patch(+ "backend.blocks.llm.llm_call", return_value=mock_response_valid+ ), patch.object(+ SmartDecisionMakerBlock,+ "_create_function_signature",+ return_value=mock_tool_functions,+ ):++ input_data = SmartDecisionMakerBlock.Input(+ prompt="Search for keywords",+ model=llm_module.LlmModel.GPT4O,+ credentials=llm_module.TEST_CREDENTIALS_INPUT, # type: ignore+ )++ # Should succeed - optional parameter missing is OK+ outputs = {}+ async for output_name, output_data in block.run(+ input_data,+ credentials=llm_module.TEST_CREDENTIALS,+ graph_id="test-graph-id",+ node_id="test-node-id",+ graph_exec_id="test-exec-id",+ node_exec_id="test-node-exec-id",+ user_id="test-user-id",+ ):+ outputs[output_name] = output_data++ # Verify tool outputs were generated correctly+ assert "tools_^_search_keywords_~_query" in outputs+ assert outputs["tools_^_search_keywords_~_query"] == "test"+ assert "tools_^_search_keywords_~_max_keyword_difficulty" in outputs+ assert outputs["tools_^_search_keywords_~_max_keyword_difficulty"] == 50+ # Optional parameter should be None when not provided+ assert "tools_^_search_keywords_~_optional_param" in outputs+ assert outputs["tools_^_search_keywords_~_optional_param"] is None++ # Test case 4: Valid tool call with ALL parameters (should succeed)+ mock_tool_call_all_params = MagicMock()+ mock_tool_call_all_params.function.name = "search_keywords"+ mock_tool_call_all_params.function.arguments = '{"query": "test", "max_keyword_difficulty": 50, "optional_param": "custom_value"}'++ mock_response_all_params = MagicMock()+ mock_response_all_params.response = None+ mock_response_all_params.tool_calls = [mock_tool_call_all_params]+ mock_response_all_params.prompt_tokens = 50+ mock_response_all_params.completion_tokens = 25+ mock_response_all_params.reasoning = None+ mock_response_all_params.raw_response = {"role": "assistant", "content": None}++ with patch(+ "backend.blocks.llm.llm_call", return_value=mock_response_all_params+ ), patch.object(+ SmartDecisionMakerBlock,+ "_create_function_signature",+ return_value=mock_tool_functions,+ ):++ input_data = SmartDecisionMakerBlock.Input(+ prompt="Search for keywords",+ model=llm_module.LlmModel.GPT4O,+ credentials=llm_module.TEST_CREDENTIALS_INPUT, # type: ignore+ )++ # Should succeed with all parameters+ outputs = {}+ async for output_name, output_data in block.run(+ input_data,+ credentials=llm_module.TEST_CREDENTIALS,+ graph_id="test-graph-id",+ node_id="test-node-id",+ graph_exec_id="test-exec-id",+ node_exec_id="test-node-exec-id",+ user_id="test-user-id",+ ):+ outputs[output_name] = output_data++ # Verify all tool outputs were generated correctly+ assert outputs["tools_^_search_keywords_~_query"] == "test"+ assert outputs["tools_^_search_keywords_~_max_keyword_difficulty"] == 50+ assert outputs["tools_^_search_keywords_~_optional_param"] == "custom_value"diff --git a/autogpt_platform/backend/load-tests/README.md b/autogpt_platform/backend/load-tests/README.mdindex ba4d62e844..1a19b02ab6 100644--- a/autogpt_platform/backend/load-tests/README.md+++ b/autogpt_platform/backend/load-tests/README.md@@ -15,19 +15,16 @@ node generate-tokens.js --count=160export K6_CLOUD_TOKEN="your-k6-cloud-token"export K6_CLOUD_PROJECT_ID="4254406"-# 4. Verify setup and run quick test-node run-tests.js verify+# 4. Run orchestrated load tests locally+node orchestrator/orchestrator.js DEV local-# 5. Run tests locally (development/debugging)-node run-tests.js run all DEV--# 6. Run tests in k6 cloud (performance testing)-node run-tests.js cloud all DEV+# 5. Run orchestrated load tests in k6 cloud (recommended)+node orchestrator/orchestrator.js DEV cloud```-## π Unified Test Runner+## π Load Test Orchestrator-The AutoGPT Platform uses a single unified test runner (`run-tests.js`) for both local and cloud execution:+The AutoGPT Platform uses a comprehensive load test orchestrator (`orchestrator/orchestrator.js`) that runs 12 optimized tests with maximum VU counts:### Available Tests@@ -60,38 +57,26 @@ The AutoGPT Platform uses a single unified test runner (`run-tests.js`) for both### Basic Commands```bash-# List available tests and show cloud credentials status-node run-tests.js list--# Quick setup verification-node run-tests.js verify--# Run specific test locally-node run-tests.js run core-api-test DEV--# Run multiple tests sequentially (comma-separated)-node run-tests.js run connectivity-test,core-api-test,marketplace-public-test DEV+# Run 12 optimized tests locally (for debugging)+node orchestrator/orchestrator.js DEV local-# Run all tests locally-node run-tests.js run all DEV+# Run 12 optimized tests in k6 cloud (recommended for performance testing)+node orchestrator/orchestrator.js DEV cloud-# Run specific test in k6 cloud-node run-tests.js cloud core-api-test DEV+# Run against production (coordinate with team!)+node orchestrator/orchestrator.js PROD cloud-# Run all tests in k6 cloud-node run-tests.js cloud all DEV+# Run individual test directly with k6+K6_ENVIRONMENT=DEV VUS=100 DURATION=3m k6 run tests/api/core-api-test.js```### NPM Scripts```bash-# Quick verification-npm run verify+# Run orchestrator locally+npm run local-# Run all tests locally-npm test--# Run all tests in k6 cloud+# Run orchestrator in k6 cloudnpm run cloud```@@ -230,8 +215,8 @@ node generate-tokens.js --count=160export K6_CLOUD_TOKEN="your-k6-cloud-token"export K6_CLOUD_PROJECT_ID="4254406" # AutoGPT Platform project ID-# Verify credentials work-node run-tests.js list # Shows β k6 cloud credentials configured+# Verify credentials work by running orchestrator+node orchestrator/orchestrator.js DEV cloud```## π File Structure@@ -239,9 +224,10 @@ node run-tests.js list # Shows β k6 cloud credentials configured```load-tests/βββ README.md # This documentation-βββ run-tests.js # Unified test runner (MAIN ENTRY POINT)-βββ generate-tokens.js # Generate pre-auth tokens+βββ generate-tokens.js # Generate pre-auth tokens (MAIN TOKEN SETUP)βββ package.json # Node.js dependencies and scripts+βββ orchestrator/+β βββ orchestrator.js # Main test orchestrator (MAIN ENTRY POINT)βββ configs/β βββ environment.js # Environment URLs and configurationβ βββ pre-authenticated-tokens.js # Generated tokens (gitignored)@@ -257,21 +243,19 @@ load-tests/β β βββ library-access-test.js # Authenticated marketplace/libraryβ βββ comprehensive/β βββ platform-journey-test.js # Complete user journey simulation-βββ orchestrator/-β βββ comprehensive-orchestrator.js # Full 25-test orchestration suiteβββ results/ # Local test results (auto-created)-βββ k6-cloud-results.txt # Cloud test URLs (auto-created)-βββ *.json # Test output files (auto-created)+βββ unified-results-*.json # Orchestrator results (auto-created)+βββ *.log # Test execution logs (auto-created)```## π― Best Practices-1. **Start with Verification**: Always run `node run-tests.js verify` first-2. **Local for Development**: Use `run` command for debugging and development-3. **Cloud for Performance**: Use `cloud` command for actual performance testing+1. **Generate Tokens First**: Always run `node generate-tokens.js --count=160` before testing+2. **Local for Development**: Use `DEV local` for debugging and development+3. **Cloud for Performance**: Use `DEV cloud` for actual performance testing4. **Monitor Real-Time**: Check k6 cloud dashboards during test execution5. **Regenerate Tokens**: Refresh tokens every 24 hours when they expire-6. **Sequential Testing**: Use comma-separated tests for organized execution+6. **Unified Testing**: Orchestrator runs 12 optimized tests automatically## π Advanced Usagediff --git a/autogpt_platform/backend/load-tests/orchestrator/comprehensive-orchestrator.js b/autogpt_platform/backend/load-tests/orchestrator/comprehensive-orchestrator.jsdeleted file mode 100644index 1fc2e75268..0000000000--- a/autogpt_platform/backend/load-tests/orchestrator/comprehensive-orchestrator.js+++ /dev/null@@ -1,611 +0,0 @@-#!/usr/bin/env node--// AutoGPT Platform Load Test Orchestrator-// Runs comprehensive test suite locally or in k6 cloud-// Collects URLs, statistics, and generates reports--const { spawn } = require("child_process");-const fs = require("fs");-const path = require("path");--console.log("π― AUTOGPT PLATFORM LOAD TEST ORCHESTRATOR\n");-console.log("===========================================\n");--// Parse command line arguments-const args = process.argv.slice(2);-const environment = args[0] || "DEV"; // LOCAL, DEV, PROD-const executionMode = args[1] || "cloud"; // local, cloud-const testScale = args[2] || "full"; // small, full--console.log(`π Target Environment: ${environment}`);-console.log(`π Execution Mode: ${executionMode}`);-console.log(`π Test Scale: ${testScale}`);--// Test scenario definitions-const testScenarios = {- // Small scale for validation (3 tests, ~5 minutes)- small: [- {- name: "Basic_Connectivity_Test",- file: "tests/basic/connectivity-test.js",- vus: 5,- duration: "30s",- },- {- name: "Core_API_Quick_Test",- file: "tests/api/core-api-test.js",- vus: 10,- duration: "1m",- },- {- name: "Marketplace_Quick_Test",- file: "tests/marketplace/public-access-test.js",- vus: 15,- duration: "1m",- },- ],-- // Full comprehensive test suite (25 tests, ~2 hours)- full: [- // Marketplace Viewing Tests- {- name: "Viewing_Marketplace_Logged_Out_Day1",- file: "tests/marketplace/public-access-test.js",- vus: 106,- duration: "3m",- },- {- name: "Viewing_Marketplace_Logged_Out_VeryHigh",- file: "tests/marketplace/public-access-test.js",- vus: 314,- duration: "3m",- },- {- name: "Viewing_Marketplace_Logged_In_Day1",- file: "tests/marketplace/library-access-test.js",- vus: 53,- duration: "3m",- },- {- name: "Viewing_Marketplace_Logged_In_VeryHigh",- file: "tests/marketplace/library-access-test.js",- vus: 157,- duration: "3m",- },-- // Library Management Tests- {- name: "Adding_Agent_to_Library_Day1",- file: "tests/marketplace/library-access-test.js",- vus: 32,- duration: "3m",- },- {- name: "Adding_Agent_to_Library_VeryHigh",- file: "tests/marketplace/library-access-test.js",- vus: 95,- duration: "3m",- },- {- name: "Viewing_Library_Home_0_Agents_Day1",- file: "tests/marketplace/library-access-test.js",- vus: 53,- duration: "3m",- },- {- name: "Viewing_Library_Home_0_Agents_VeryHigh",- file: "tests/marketplace/library-access-test.js",- vus: 157,- duration: "3m",- },-- // Core API Tests- {- name: "Core_API_Load_Test",- file: "tests/api/core-api-test.js",- vus: 100,- duration: "3m",- },- {- name: "Graph_Execution_Load_Test",- file: "tests/api/graph-execution-test.js",- vus: 100,- duration: "3m",- },-- // Single API Endpoint Tests- {- name: "Credits_API_Single_Endpoint",- file: "tests/basic/single-endpoint-test.js",- vus: 50,- duration: "3m",- env: { ENDPOINT: "credits", CONCURRENT_REQUESTS: 10 },- },- {- name: "Graphs_API_Single_Endpoint",- file: "tests/basic/single-endpoint-test.js",- vus: 50,- duration: "3m",- env: { ENDPOINT: "graphs", CONCURRENT_REQUESTS: 10 },- },- {- name: "Blocks_API_Single_Endpoint",- file: "tests/basic/single-endpoint-test.js",- vus: 50,- duration: "3m",- env: { ENDPOINT: "blocks", CONCURRENT_REQUESTS: 10 },- },- {- name: "Executions_API_Single_Endpoint",- file: "tests/basic/single-endpoint-test.js",- vus: 50,- duration: "3m",- env: { ENDPOINT: "executions", CONCURRENT_REQUESTS: 10 },- },-- // Comprehensive Platform Tests- {- name: "Comprehensive_Platform_Low",- file: "tests/comprehensive/platform-journey-test.js",- vus: 25,- duration: "3m",- },- {- name: "Comprehensive_Platform_Medium",- file: "tests/comprehensive/platform-journey-test.js",- vus: 50,- duration: "3m",- },- {- name: "Comprehensive_Platform_High",- file: "tests/comprehensive/platform-journey-test.js",- vus: 100,- duration: "3m",- },-- // User Authentication Workflows- {- name: "User_Auth_Workflows_Day1",- file: "tests/basic/connectivity-test.js",- vus: 50,- duration: "3m",- },- {- name: "User_Auth_Workflows_VeryHigh",- file: "tests/basic/connectivity-test.js",- vus: 100,- duration: "3m",- },-- // Mixed Load Tests- {- name: "Mixed_Load_Light",- file: "tests/api/core-api-test.js",- vus: 75,- duration: "5m",- },- {- name: "Mixed_Load_Heavy",- file: "tests/marketplace/public-access-test.js",- vus: 200,- duration: "5m",- },-- // Stress Tests- {- name: "Marketplace_Stress_Test",- file: "tests/marketplace/public-access-test.js",- vus: 500,- duration: "3m",- },- {- name: "Core_API_Stress_Test",- file: "tests/api/core-api-test.js",- vus: 300,- duration: "3m",- },-- // Extended Duration Tests- {- name: "Long_Duration_Marketplace",- file: "tests/marketplace/library-access-test.js",- vus: 100,- duration: "10m",- },- {- name: "Long_Duration_Core_API",- file: "tests/api/core-api-test.js",- vus: 100,- duration: "10m",- },- ],-};--const scenarios = testScenarios[testScale];-console.log(`π Running ${scenarios.length} test scenarios`);--// Results collection-const results = [];-const cloudUrls = [];-const detailedMetrics = [];--// Create results directory-const timestamp = new Date()- .toISOString()- .replace(/[:.]/g, "-")- .substring(0, 16);-const resultsDir = `results-${environment.toLowerCase()}-${executionMode}-${testScale}-${timestamp}`;-if (!fs.existsSync(resultsDir)) {- fs.mkdirSync(resultsDir);-}--// Function to run a single test-function runTest(scenario, testIndex) {- return new Promise((resolve, reject) => {- console.log(`\nπ Test ${testIndex}/${scenarios.length}: ${scenario.name}`);- console.log(- `π Config: ${scenario.vus} VUs Γ ${scenario.duration} (${executionMode} mode)`,- );- console.log(`π Script: ${scenario.file}`);-- // Build k6 command- let k6Command, k6Args;-- // Determine k6 binary location- const isInPod = fs.existsSync("/app/k6-v0.54.0-linux-amd64/k6");- const k6Binary = isInPod ? "/app/k6-v0.54.0-linux-amd64/k6" : "k6";-- // Build environment variables- const envVars = [- `K6_ENVIRONMENT=${environment}`,- `VUS=${scenario.vus}`,- `DURATION=${scenario.duration}`,- `RAMP_UP=30s`,- `RAMP_DOWN=30s`,- `THRESHOLD_P95=60000`,- `THRESHOLD_P99=60000`,- ];-- // Add scenario-specific environment variables- if (scenario.env) {- Object.keys(scenario.env).forEach((key) => {- envVars.push(`${key}=${scenario.env[key]}`);- });- }-- // Configure command based on execution mode- if (executionMode === "cloud") {- k6Command = k6Binary;- k6Args = ["cloud", "run", scenario.file];- // Add environment variables as --env flags- envVars.forEach((env) => {- k6Args.push("--env", env);- });- } else {- k6Command = k6Binary;- k6Args = ["run", scenario.file];-- // Add local output files- const outputFile = path.join(resultsDir, `${scenario.name}.json`);- const summaryFile = path.join(- resultsDir,- `${scenario.name}_summary.json`,- );- k6Args.push("--out", `json=${outputFile}`);- k6Args.push("--summary-export", summaryFile);- }-- const startTime = Date.now();- let testUrl = "";- let stdout = "";- let stderr = "";-- console.log(`β±οΈ Test started: ${new Date().toISOString()}`);-- // Set environment variables for spawned process- const processEnv = { ...process.env };- envVars.forEach((env) => {- const [key, value] = env.split("=");- processEnv[key] = value;- });-- const childProcess = spawn(k6Command, k6Args, {- env: processEnv,- stdio: ["ignore", "pipe", "pipe"],- });-- // Handle stdout- childProcess.stdout.on("data", (data) => {- const output = data.toString();- stdout += output;-- // Extract k6 cloud URL- if (executionMode === "cloud") {- const urlMatch = output.match(/output:\s*(https:\/\/[^\s]+)/);- if (urlMatch) {- testUrl = urlMatch[1];- console.log(`π Test URL: ${testUrl}`);- }- }-- // Show progress indicators- if (output.includes("Run [")) {- const progressMatch = output.match(/Run\s+\[\s*(\d+)%\s*\]/);- if (progressMatch) {- process.stdout.write(`\rβ³ Progress: ${progressMatch[1]}%`);- }- }- });-- // Handle stderr- childProcess.stderr.on("data", (data) => {- stderr += data.toString();- });-- // Handle process completion- childProcess.on("close", (code) => {- const endTime = Date.now();- const duration = Math.round((endTime - startTime) / 1000);-- console.log(`\nβ±οΈ Completed in ${duration}s`);-- if (code === 0) {- console.log(`β ${scenario.name} SUCCESS`);-- const result = {- test: scenario.name,- status: "SUCCESS",- duration: `${duration}s`,- vus: scenario.vus,- target_duration: scenario.duration,- url: testUrl || "N/A",- execution_mode: executionMode,- environment: environment,- completed_at: new Date().toISOString(),- };-- results.push(result);-- if (testUrl) {- cloudUrls.push(`${scenario.name}: ${testUrl}`);- }-- // Store detailed output for analysis- detailedMetrics.push({- test: scenario.name,- stdout_lines: stdout.split("\n").length,- stderr_lines: stderr.split("\n").length,- has_url: !!testUrl,- });-- resolve(result);- } else {- console.error(`β ${scenario.name} FAILED (exit code ${code})`);-- const result = {- test: scenario.name,- status: "FAILED",- error: `Exit code ${code}`,- duration: `${duration}s`,- vus: scenario.vus,- execution_mode: executionMode,- environment: environment,- completed_at: new Date().toISOString(),- };-- results.push(result);- reject(new Error(`Test failed with exit code ${code}`));- }- });-- // Handle spawn errors- childProcess.on("error", (error) => {- console.error(`β ${scenario.name} ERROR:`, error.message);-- results.push({- test: scenario.name,- status: "ERROR",- error: error.message,- execution_mode: executionMode,- environment: environment,- });-- reject(error);- });- });-}--// Main orchestration function-async function runOrchestrator() {- const estimatedMinutes = scenarios.length * (testScale === "small" ? 2 : 5);- console.log(`\nπ― Starting ${testScale} test suite on ${environment}`);- console.log(`π Estimated time: ~${estimatedMinutes} minutes`);- console.log(`π©οΈ Execution: ${executionMode} mode\n`);-- const startTime = Date.now();- let successCount = 0;- let failureCount = 0;-- // Run tests sequentially- for (let i = 0; i < scenarios.length; i++) {- try {- await runTest(scenarios[i], i + 1);- successCount++;-- // Pause between tests (avoid overwhelming k6 cloud API)- if (i < scenarios.length - 1) {- const pauseSeconds = testScale === "small" ? 10 : 30;- console.log(`\nβΈοΈ Pausing ${pauseSeconds}s before next test...\n`);- await new Promise((resolve) =>- setTimeout(resolve, pauseSeconds * 1000),- );- }- } catch (error) {- failureCount++;- console.log(`π₯ Continuing after failure...\n`);-- // Brief pause before continuing- if (i < scenarios.length - 1) {- await new Promise((resolve) => setTimeout(resolve, 15000));- }- }- }-- const totalTime = Math.round((Date.now() - startTime) / 1000);- await generateReports(successCount, failureCount, totalTime);-}--// Generate comprehensive reports-async function generateReports(successCount, failureCount, totalTime) {- console.log("\nπ LOAD TEST ORCHESTRATOR COMPLETE\n");- console.log("===================================\n");-- // Summary statistics- const successRate = Math.round((successCount / scenarios.length) * 100);- console.log("π EXECUTION SUMMARY:");- console.log(- `β Successful tests: ${successCount}/${scenarios.length} (${successRate}%)`,- );- console.log(`β Failed tests: ${failureCount}/${scenarios.length}`);- console.log(`β±οΈ Total execution time: ${Math.round(totalTime / 60)} minutes`);- console.log(`π Environment: ${environment}`);- console.log(`π Mode: ${executionMode}`);-- // Generate CSV report- const csvHeaders =- "Test Name,Status,VUs,Target Duration,Actual Duration,Environment,Mode,Test URL,Error,Completed At";- const csvRows = results.map(- (r) =>- `"${r.test}","${r.status}",${r.vus},"${r.target_duration || "N/A"}","${r.duration || "N/A"}","${r.environment}","${r.execution_mode}","${r.url || "N/A"}","${r.error || "None"}","${r.completed_at || "N/A"}"`,- );-- const csvContent = [csvHeaders, ...csvRows].join("\n");- const csvFile = path.join(resultsDir, "orchestrator_results.csv");- fs.writeFileSync(csvFile, csvContent);- console.log(`\nπ CSV Report: ${csvFile}`);-- // Generate cloud URLs file- if (executionMode === "cloud" && cloudUrls.length > 0) {- const urlsContent = [- `# AutoGPT Platform Load Test URLs`,- `# Environment: ${environment}`,- `# Generated: ${new Date().toISOString()}`,- `# Dashboard: https://significantgravitas.grafana.net/a/k6-app/`,- "",- ...cloudUrls,- "",- "# Direct Dashboard Access:",- "https://significantgravitas.grafana.net/a/k6-app/",- ].join("\n");-- const urlsFile = path.join(resultsDir, "cloud_test_urls.txt");- fs.writeFileSync(urlsFile, urlsContent);- console.log(`π Cloud URLs: ${urlsFile}`);- }-- // Generate detailed JSON report- const jsonReport = {- meta: {- orchestrator_version: "1.0",- environment: environment,- execution_mode: executionMode,- test_scale: testScale,- total_scenarios: scenarios.length,- generated_at: new Date().toISOString(),- results_directory: resultsDir,- },- summary: {- successful_tests: successCount,- failed_tests: failureCount,- success_rate: `${successRate}%`,- total_execution_time_seconds: totalTime,- total_execution_time_minutes: Math.round(totalTime / 60),- },- test_results: results,- detailed_metrics: detailedMetrics,- cloud_urls: cloudUrls,- };-- const jsonFile = path.join(resultsDir, "orchestrator_results.json");- fs.writeFileSync(jsonFile, JSON.stringify(jsonReport, null, 2));- console.log(`π JSON Report: ${jsonFile}`);-- // Display immediate results- if (executionMode === "cloud" && cloudUrls.length > 0) {- console.log("\nπ K6 CLOUD TEST DASHBOARD URLS:");- console.log("================================");- cloudUrls.slice(0, 5).forEach((url) => console.log(url));- if (cloudUrls.length > 5) {- console.log(`... and ${cloudUrls.length - 5} more URLs in ${urlsFile}`);- }- console.log(- "\nπ Main Dashboard: https://significantgravitas.grafana.net/a/k6-app/",- );- }-- console.log(`\nπ All results saved in: ${resultsDir}/`);- console.log("π Load Test Orchestrator finished successfully!");-}--// Show usage help-function showUsage() {- console.log("π― AutoGPT Platform Load Test Orchestrator\n");- console.log(- "Usage: node load-test-orchestrator.js [ENVIRONMENT] [MODE] [SCALE]\n",- );- console.log("ENVIRONMENT:");- console.log(" LOCAL - http://localhost:8006 (local development)");- console.log(" DEV - https://dev-api.agpt.co (development server)");- console.log(- " PROD - https://api.agpt.co (production - coordinate with team!)\n",- );- console.log("MODE:");- console.log(" local - Run locally with JSON output files");- console.log(" cloud - Run in k6 cloud with dashboard monitoring\n");- console.log("SCALE:");- console.log(" small - 3 validation tests (~5 minutes)");- console.log(" full - 25 comprehensive tests (~2 hours)\n");- console.log("Examples:");- console.log(" node load-test-orchestrator.js DEV cloud small");- console.log(" node load-test-orchestrator.js LOCAL local small");- console.log(" node load-test-orchestrator.js DEV cloud full");- console.log(- " node load-test-orchestrator.js PROD cloud full # Coordinate with team!\n",- );- console.log("Requirements:");- console.log(- " - Pre-authenticated tokens generated (node generate-tokens.js)",- );- console.log(" - k6 installed locally or run from Kubernetes pod");- console.log(" - For cloud mode: K6_CLOUD_TOKEN and K6_CLOUD_PROJECT_ID set");-}--// Handle command line help-if (args.includes("--help") || args.includes("-h")) {- showUsage();- process.exit(0);-}--// Handle graceful shutdown-process.on("SIGINT", () => {- console.log("\nπ Orchestrator interrupted by user");- console.log("π Generating partial results...");- generateReports(- results.filter((r) => r.status === "SUCCESS").length,- results.filter((r) => r.status === "FAILED").length,- 0,- ).then(() => {- console.log("πββοΈ Partial results saved");- process.exit(0);- });-});--// Start orchestrator-if (require.main === module) {- runOrchestrator().catch((error) => {- console.error("π₯ Orchestrator failed:", error);- process.exit(1);- });-}--module.exports = { runOrchestrator, testScenarios };diff --git a/autogpt_platform/backend/load-tests/orchestrator/orchestrator.js b/autogpt_platform/backend/load-tests/orchestrator/orchestrator.jsnew file mode 100644index 0000000000..e7a12ddd66--- /dev/null+++ b/autogpt_platform/backend/load-tests/orchestrator/orchestrator.js@@ -0,0 +1,362 @@+#!/usr/bin/env node++/**+ * AutoGPT Platform Load Test Orchestrator+ *+ * Optimized test suite with only the highest VU count for each unique test type.+ * Eliminates duplicate tests and focuses on maximum load testing.+ */++import { spawn } from 'child_process';+import fs from 'fs';++console.log("π― AUTOGPT PLATFORM LOAD TEST ORCHESTRATOR\n");+console.log("===========================================\n");++// Parse command line arguments+const args = process.argv.slice(2);+const environment = args[0] || "DEV"; // LOCAL, DEV, PROD+const executionMode = args[1] || "cloud"; // local, cloud++console.log(`π Target Environment: ${environment}`);+console.log(`π Execution Mode: ${executionMode}`);++// Unified test scenarios - only highest VUs for each unique test+const unifiedTestScenarios = [+ // 1. Marketplace Public Access (highest VUs: 314)+ {+ name: "Marketplace_Public_Access_Max_Load",+ file: "tests/marketplace/public-access-test.js",+ vus: 314,+ duration: "3m",+ rampUp: "30s",+ rampDown: "30s",+ description: "Public marketplace browsing at maximum load"+ },++ // 2. Marketplace Authenticated Access (highest VUs: 157)+ {+ name: "Marketplace_Authenticated_Access_Max_Load",+ file: "tests/marketplace/library-access-test.js",+ vus: 157,+ duration: "3m",+ rampUp: "30s",+ rampDown: "30s",+ description: "Authenticated marketplace/library operations at maximum load"+ },++ // 3. Core API Load Test (highest VUs: 100)+ {+ name: "Core_API_Max_Load",+ file: "tests/api/core-api-test.js",+ vus: 100,+ duration: "5m",+ rampUp: "1m",+ rampDown: "1m",+ description: "Core authenticated API endpoints at maximum load"+ },++ // 4. Graph Execution Load Test (highest VUs: 100)+ {+ name: "Graph_Execution_Max_Load",+ file: "tests/api/graph-execution-test.js",+ vus: 100,+ duration: "5m",+ rampUp: "1m",+ rampDown: "1m",+ description: "Graph workflow execution pipeline at maximum load"+ },++ // 5. Credits API Single Endpoint (upgraded to 100 VUs)+ {+ name: "Credits_API_Max_Load",+ file: "tests/basic/single-endpoint-test.js",+ vus: 100,+ duration: "3m",+ rampUp: "30s",+ rampDown: "30s",+ env: { ENDPOINT: "credits", CONCURRENT_REQUESTS: "1" },+ description: "Credits API endpoint at maximum load"+ },++ // 6. Graphs API Single Endpoint (upgraded to 100 VUs)+ {+ name: "Graphs_API_Max_Load",+ file: "tests/basic/single-endpoint-test.js",+ vus: 100,+ duration: "3m",+ rampUp: "30s",+ rampDown: "30s",+ env: { ENDPOINT: "graphs", CONCURRENT_REQUESTS: "1" },+ description: "Graphs API endpoint at maximum load"+ },++ // 7. Blocks API Single Endpoint (upgraded to 100 VUs)+ {+ name: "Blocks_API_Max_Load",+ file: "tests/basic/single-endpoint-test.js",+ vus: 100,+ duration: "3m",+ rampUp: "30s",+ rampDown: "30s",+ env: { ENDPOINT: "blocks", CONCURRENT_REQUESTS: "1" },+ description: "Blocks API endpoint at maximum load"+ },++ // 8. Executions API Single Endpoint (upgraded to 100 VUs)+ {+ name: "Executions_API_Max_Load",+ file: "tests/basic/single-endpoint-test.js",+ vus: 100,+ duration: "3m",+ rampUp: "30s",+ rampDown: "30s",+ env: { ENDPOINT: "executions", CONCURRENT_REQUESTS: "1" },+ description: "Executions API endpoint at maximum load"+ },++ // 9. Comprehensive Platform Journey (highest VUs: 100)+ {+ name: "Comprehensive_Platform_Max_Load",+ file: "tests/comprehensive/platform-journey-test.js",+ vus: 100,+ duration: "3m",+ rampUp: "30s",+ rampDown: "30s",+ description: "End-to-end user journey simulation at maximum load"+ },++ // 10. Marketplace Stress Test (highest VUs: 500)+ {+ name: "Marketplace_Stress_Test",+ file: "tests/marketplace/public-access-test.js",+ vus: 500,+ duration: "2m",+ rampUp: "1m",+ rampDown: "1m",+ description: "Ultimate marketplace stress test"+ },++ // 11. Core API Stress Test (highest VUs: 500)+ {+ name: "Core_API_Stress_Test",+ file: "tests/api/core-api-test.js",+ vus: 500,+ duration: "2m",+ rampUp: "1m",+ rampDown: "1m",+ description: "Ultimate core API stress test"+ },++ // 12. Long Duration Core API Test (highest VUs: 100, longest duration)+ {+ name: "Long_Duration_Core_API_Test",+ file: "tests/api/core-api-test.js",+ vus: 100,+ duration: "10m",+ rampUp: "1m",+ rampDown: "1m",+ description: "Extended duration core API endurance test"+ }+];++// Configuration+const K6_CLOUD_TOKEN = process.env.K6_CLOUD_TOKEN || '9347b8bd716cadc243e92f7d2f89107febfb81b49f2340d17da515d7b0513b51';+const K6_CLOUD_PROJECT_ID = process.env.K6_CLOUD_PROJECT_ID || '4254406';+const PAUSE_BETWEEN_TESTS = 30; // seconds++/**+ * Sleep for specified milliseconds+ */+function sleep(ms) {+ return new Promise(resolve => setTimeout(resolve, ms));+}++/**+ * Run a single k6 test+ */+async function runTest(test, index) {+ return new Promise((resolve, reject) => {+ console.log(`\nπ Test ${index + 1}/${unifiedTestScenarios.length}: ${test.name}`);+ console.log(`π Config: ${test.vus} VUs Γ ${test.duration} (${executionMode} mode)`);+ console.log(`π Script: ${test.file}`);+ console.log(`π Description: ${test.description}`);+ console.log(`β±οΈ Test started: ${new Date().toISOString()}`);++ const env = {+ K6_CLOUD_TOKEN,+ K6_CLOUD_PROJECT_ID,+ K6_ENVIRONMENT: environment,+ VUS: test.vus.toString(),+ DURATION: test.duration,+ RAMP_UP: test.rampUp,+ RAMP_DOWN: test.rampDown,+ ...test.env+ };++ let args;+ if (executionMode === 'cloud') {+ args = [+ 'cloud', 'run',+ ...Object.entries(env).map(([key, value]) => ['--env', `${key}=${value}`]).flat(),+ test.file+ ];+ } else {+ args = [+ 'run',+ ...Object.entries(env).map(([key, value]) => ['--env', `${key}=${value}`]).flat(),+ test.file+ ];+ }++ const k6Process = spawn('k6', args, {+ stdio: ['ignore', 'pipe', 'pipe'],+ env: { ...process.env, ...env }+ });++ let output = '';+ let testId = null;++ k6Process.stdout.on('data', (data) => {+ const str = data.toString();+ output += str;++ // Extract test ID from k6 cloud output+ const testIdMatch = str.match(/Test created: .*\/(\d+)/);+ if (testIdMatch) {+ testId = testIdMatch[1];+ console.log(`π Test URL: https://significantgravitas.grafana.net/a/k6-app/runs/${testId}`);+ }++ // Show progress updates+ const progressMatch = str.match(/(\d+)%/);+ if (progressMatch) {+ process.stdout.write(`\rβ³ Progress: ${progressMatch[1]}%`);+ }+ });++ k6Process.stderr.on('data', (data) => {+ output += data.toString();+ });++ k6Process.on('close', (code) => {+ process.stdout.write('\n'); // Clear progress line++ if (code === 0) {+ console.log(`β ${test.name} SUCCESS`);+ resolve({+ success: true,+ testId,+ url: testId ? `https://significantgravitas.grafana.net/a/k6-app/runs/${testId}` : 'unknown',+ vus: test.vus,+ duration: test.duration+ });+ } else {+ console.log(`β ${test.name} FAILED (exit code ${code})`);+ resolve({+ success: false,+ testId,+ url: testId ? `https://significantgravitas.grafana.net/a/k6-app/runs/${testId}` : 'unknown',+ exitCode: code,+ vus: test.vus,+ duration: test.duration+ });+ }+ });++ k6Process.on('error', (error) => {+ console.log(`β ${test.name} ERROR: ${error.message}`);+ reject(error);+ });+ });+}++/**+ * Main execution+ */+async function main() {+ console.log(`\nπ UNIFIED TEST PLAN`);+ console.log(`π Total tests: ${unifiedTestScenarios.length} (reduced from 25 original tests)`);+ console.log(`β±οΈ Estimated duration: ~60 minutes\n`);++ console.log(`π Test Summary:`);+ unifiedTestScenarios.forEach((test, i) => {+ console.log(` ${i + 1}. ${test.name} (${test.vus} VUs Γ ${test.duration})`);+ });+ console.log('');++ const results = [];++ for (let i = 0; i < unifiedTestScenarios.length; i++) {+ const test = unifiedTestScenarios[i];++ try {+ const result = await runTest(test, i);+ results.push({ ...test, ...result });++ // Pause between tests (except after the last one)+ if (i < unifiedTestScenarios.length - 1) {+ console.log(`\nβΈοΈ Pausing ${PAUSE_BETWEEN_TESTS}s before next test...`);+ await sleep(PAUSE_BETWEEN_TESTS * 1000);+ }+ } catch (error) {+ console.error(`π₯ Fatal error running ${test.name}:`, error.message);+ results.push({ ...test, success: false, error: error.message });+ }+ }++ // Summary+ console.log('\n' + '='.repeat(60));+ console.log('π UNIFIED LOAD TEST RESULTS SUMMARY');+ console.log('='.repeat(60));++ const successful = results.filter(r => r.success);+ const failed = results.filter(r => !r.success);++ console.log(`β Successful tests: ${successful.length}/${results.length} (${Math.round(successful.length / results.length * 100)}%)`);+ console.log(`β Failed tests: ${failed.length}/${results.length}`);++ if (successful.length > 0) {+ console.log('\nβ SUCCESSFUL TESTS:');+ successful.forEach(test => {+ console.log(` β’ ${test.name} (${test.vus} VUs) - ${test.url}`);+ });+ }++ if (failed.length > 0) {+ console.log('\nβ FAILED TESTS:');+ failed.forEach(test => {+ console.log(` β’ ${test.name} (${test.vus} VUs) - ${test.url || 'no URL'} (exit: ${test.exitCode || 'unknown'})`);+ });+ }++ // Calculate total VU-minutes tested+ const totalVuMinutes = results.reduce((sum, test) => {+ const minutes = parseFloat(test.duration.replace(/[ms]/g, ''));+ const multiplier = test.duration.includes('m') ? 1 : (1/60); // convert seconds to minutes+ return sum + (test.vus * minutes * multiplier);+ }, 0);++ console.log(`\nπ LOAD TESTING SUMMARY:`);+ console.log(` β’ Total VU-minutes tested: ${Math.round(totalVuMinutes)}`);+ console.log(` β’ Peak concurrent VUs: ${Math.max(...results.map(r => r.vus))}`);+ console.log(` β’ Average test duration: ${(results.reduce((sum, r) => sum + parseFloat(r.duration.replace(/[ms]/g, '')), 0) / results.length).toFixed(1)}${results[0].duration.includes('m') ? 'm' : 's'}`);++ // Write results to file+ const timestamp = Math.floor(Date.now() / 1000);+ const resultsFile = `unified-results-${timestamp}.json`;+ fs.writeFileSync(resultsFile, JSON.stringify(results, null, 2));+ console.log(`\nπ Detailed results saved to: ${resultsFile}`);++ console.log(`\nπ UNIFIED LOAD TEST ORCHESTRATOR COMPLETE\n`);++ process.exit(failed.length === 0 ? 0 : 1);+}++// Run if called directly+if (process.argv[1] === new URL(import.meta.url).pathname) {+ main().catch(error => {+ console.error('π₯ Fatal error:', error);+ process.exit(1);+ });+}\ No newline at end of filediff --git a/autogpt_platform/backend/load-tests/run-tests.js b/autogpt_platform/backend/load-tests/run-tests.jsdeleted file mode 100644index 4222150e94..0000000000--- a/autogpt_platform/backend/load-tests/run-tests.js+++ /dev/null@@ -1,268 +0,0 @@-#!/usr/bin/env node-/**- * Unified Load Test Runner- *- * Supports both local execution and k6 cloud execution with the same interface.- * Automatically detects cloud credentials and provides seamless switching.- *- * Usage:- * node run-tests.js verify # Quick verification (1 VU, 10s)- * node run-tests.js run core-api-test DEV # Run specific test locally- * node run-tests.js run all DEV # Run all tests locally- * node run-tests.js cloud core-api DEV # Run specific test in k6 cloud- * node run-tests.js cloud all DEV # Run all tests in k6 cloud- */--import { execSync } from "child_process";-import fs from "fs";--const TESTS = {- "connectivity-test": {- script: "tests/basic/connectivity-test.js",- description: "Basic connectivity validation",- cloudConfig: { vus: 10, duration: "2m" },- },- "single-endpoint-test": {- script: "tests/basic/single-endpoint-test.js",- description: "Individual API endpoint testing",- cloudConfig: { vus: 25, duration: "3m" },- },- "core-api-test": {- script: "tests/api/core-api-test.js",- description: "Core API endpoints performance test",- cloudConfig: { vus: 100, duration: "5m" },- },- "graph-execution-test": {- script: "tests/api/graph-execution-test.js",- description: "Graph creation and execution pipeline test",- cloudConfig: { vus: 80, duration: "5m" },- },- "marketplace-public-test": {- script: "tests/marketplace/public-access-test.js",- description: "Public marketplace browsing test",- cloudConfig: { vus: 150, duration: "3m" },- },- "marketplace-library-test": {- script: "tests/marketplace/library-access-test.js",- description: "Authenticated marketplace/library test",- cloudConfig: { vus: 100, duration: "4m" },- },- "comprehensive-test": {- script: "tests/comprehensive/platform-journey-test.js",- description: "Complete user journey simulation",- cloudConfig: { vus: 50, duration: "6m" },- },-};--function checkCloudCredentials() {- const token = process.env.K6_CLOUD_TOKEN;- const projectId = process.env.K6_CLOUD_PROJECT_ID;-- if (!token || !projectId) {- console.log("β Missing k6 cloud credentials");- console.log("Set: K6_CLOUD_TOKEN and K6_CLOUD_PROJECT_ID");- return false;- }- return true;-}--function verifySetup() {- console.log("π Quick Setup Verification");-- // Check tokens- if (!fs.existsSync("configs/pre-authenticated-tokens.js")) {- console.log("β No tokens found. Run: node generate-tokens.js");- return false;- }-- // Quick test- try {- execSync(- "K6_ENVIRONMENT=DEV VUS=1 DURATION=10s k6 run tests/basic/connectivity-test.js --quiet",- { stdio: "inherit", cwd: process.cwd() },- );- console.log("β Verification successful");- return true;- } catch (error) {- console.log("β Verification failed");- return false;- }-}--function runLocalTest(testName, environment) {- const test = TESTS[testName];- if (!test) {- console.log(`β Unknown test: ${testName}`);- console.log("Available tests:", Object.keys(TESTS).join(", "));- return;- }-- console.log(`π Running ${test.description} locally on ${environment}`);-- try {- const cmd = `K6_ENVIRONMENT=${environment} VUS=5 DURATION=30s k6 run ${test.script}`;- execSync(cmd, { stdio: "inherit", cwd: process.cwd() });- console.log("β Test completed");- } catch (error) {- console.log("β Test failed");- }-}--function runCloudTest(testName, environment) {- const test = TESTS[testName];- if (!test) {- console.log(`β Unknown test: ${testName}`);- console.log("Available tests:", Object.keys(TESTS).join(", "));- return;- }-- const { vus, duration } = test.cloudConfig;- console.log(`βοΈ Running ${test.description} in k6 cloud`);- console.log(` Environment: ${environment}`);- console.log(` Config: ${vus} VUs Γ ${duration}`);-- try {- const cmd = `k6 cloud run --env K6_ENVIRONMENT=${environment} --env VUS=${vus} --env DURATION=${duration} --env RAMP_UP=30s --env RAMP_DOWN=30s ${test.script}`;- const output = execSync(cmd, {- stdio: "pipe",- cwd: process.cwd(),- encoding: "utf8",- });-- // Extract and display URL- const urlMatch = output.match(/https:\/\/[^\s]*grafana[^\s]*/);- if (urlMatch) {- const url = urlMatch[0];- console.log(`π Test URL: ${url}`);-- // Save to results file- const timestamp = new Date().toISOString();- const result = `${timestamp} - ${testName}: ${url}\n`;- fs.appendFileSync("k6-cloud-results.txt", result);- }-- console.log("β Cloud test started successfully");- } catch (error) {- console.log("β Cloud test failed to start");- console.log(error.message);- }-}--function runAllLocalTests(environment) {- console.log(`π Running all tests locally on ${environment}`);-- for (const [testName, test] of Object.entries(TESTS)) {- console.log(`\nπ ${test.description}`);- runLocalTest(testName, environment);- }-}--function runAllCloudTests(environment) {- console.log(`βοΈ Running all tests in k6 cloud on ${environment}`);-- const testNames = Object.keys(TESTS);- for (let i = 0; i < testNames.length; i++) {- const testName = testNames[i];- console.log(`\nπ Test ${i + 1}/${testNames.length}: ${testName}`);-- runCloudTest(testName, environment);-- // Brief pause between cloud tests (except last one)- if (i < testNames.length - 1) {- console.log("βΈοΈ Waiting 2 minutes before next cloud test...");- execSync("sleep 120");- }- }-}--function listTests() {- console.log("π Available Tests:");- console.log("==================");-- Object.entries(TESTS).forEach(([name, test]) => {- const { vus, duration } = test.cloudConfig;- console.log(` ${name.padEnd(20)} - ${test.description}`);- console.log(` ${" ".repeat(20)} Cloud: ${vus} VUs Γ ${duration}`);- });-- console.log("\nπ Available Environments: LOCAL, DEV, PROD");- console.log("\nπ‘ Examples:");- console.log(" # Local execution (5 VUs, 30s)");- console.log(" node run-tests.js verify");- console.log(" node run-tests.js run core-api-test DEV");- console.log(" node run-tests.js run core-api-test,marketplace-test DEV");- console.log(" node run-tests.js run all DEV");- console.log("");- console.log(" # Cloud execution (high VUs, longer duration)");- console.log(" node run-tests.js cloud core-api DEV");- console.log(" node run-tests.js cloud all DEV");-- const hasCloudCreds = checkCloudCredentials();- console.log(- `\nβοΈ Cloud Status: ${hasCloudCreds ? "β Configured" : "β Missing credentials"}`,- );-}--function runSequentialTests(testNames, environment, isCloud = false) {- const tests = testNames.split(",").map((t) => t.trim());- const mode = isCloud ? "cloud" : "local";- console.log(- `π Running ${tests.length} tests sequentially in ${mode} mode on ${environment}`,- );-- for (let i = 0; i < tests.length; i++) {- const testName = tests[i];- console.log(`\nπ Test ${i + 1}/${tests.length}: ${testName}`);-- if (isCloud) {- runCloudTest(testName, environment);- } else {- runLocalTest(testName, environment);- }-- // Brief pause between tests (except last one)- if (i < tests.length - 1) {- const pauseTime = isCloud ? "2 minutes" : "10 seconds";- const pauseCmd = isCloud ? "sleep 120" : "sleep 10";- console.log(`βΈοΈ Waiting ${pauseTime} before next test...`);- if (!isCloud) {- // Note: In real implementation, would use setTimeout/sleep for local tests- }- }- }-}--// Main CLI-const [, , command, testOrEnv, environment] = process.argv;--switch (command) {- case "verify":- verifySetup();- break;- case "list":- listTests();- break;- case "run":- if (testOrEnv === "all") {- runAllLocalTests(environment || "DEV");- } else if (testOrEnv?.includes(",")) {- runSequentialTests(testOrEnv, environment || "DEV", false);- } else {- runLocalTest(testOrEnv, environment || "DEV");- }- break;- case "cloud":- if (!checkCloudCredentials()) {- process.exit(1);- }- if (testOrEnv === "all") {- runAllCloudTests(environment || "DEV");- } else if (testOrEnv?.includes(",")) {- runSequentialTests(testOrEnv, environment || "DEV", true);- } else {- runCloudTest(testOrEnv, environment || "DEV");- }- break;- default:- listTests();-}