AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): Add correctness score to execution activity generation (#11325)
Zamil Majdy6037f8050297
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.pyindex 922672acd0..8025fc5d81 100644--- a/autogpt_platform/backend/backend/data/execution.py+++ b/autogpt_platform/backend/backend/data/execution.py@@ -175,6 +175,10 @@ class GraphExecutionMeta(BaseDbModel):default=None,description="AI-generated summary of what the agent did",)+ correctness_score: float | None = Field(+ default=None,+ description="AI-generated score (0.0-1.0) indicating how well the execution achieved its intended purpose",+ )def to_db(self) -> GraphExecutionStats:return GraphExecutionStats(@@ -187,6 +191,7 @@ class GraphExecutionMeta(BaseDbModel):node_error_count=self.node_error_count,error=self.error,activity_status=self.activity_status,+ correctness_score=self.correctness_score,)stats: Stats | None@@ -244,6 +249,7 @@ class GraphExecutionMeta(BaseDbModel):else stats.error),activity_status=stats.activity_status,+ correctness_score=stats.correctness_score,)if statselse Nonediff --git a/autogpt_platform/backend/backend/data/model.py b/autogpt_platform/backend/backend/data/model.pyindex ac3c85461d..1c32a9f444 100644--- a/autogpt_platform/backend/backend/data/model.py+++ b/autogpt_platform/backend/backend/data/model.py@@ -833,6 +833,10 @@ class GraphExecutionStats(BaseModel):activity_status: Optional[str] = Field(default=None, description="AI-generated summary of what the agent did")+ correctness_score: Optional[float] = Field(+ default=None,+ description="AI-generated score (0.0-1.0) indicating how well the execution achieved its intended purpose",+ )class UserExecutionSummaryStats(BaseModel):diff --git a/autogpt_platform/backend/backend/executor/activity_status_generator.py b/autogpt_platform/backend/backend/executor/activity_status_generator.pyindex b9d1ed7bd5..faa11a4410 100644--- a/autogpt_platform/backend/backend/executor/activity_status_generator.py+++ b/autogpt_platform/backend/backend/executor/activity_status_generator.py@@ -13,12 +13,11 @@ except ImportError:from pydantic import SecretStr-from backend.blocks.llm import LlmModel, llm_call+from backend.blocks.llm import AIStructuredResponseGeneratorBlock, LlmModelfrom backend.data.block import get_blockfrom backend.data.execution import ExecutionStatus, NodeExecutionResultfrom backend.data.model import APIKeyCredentials, GraphExecutionStatsfrom backend.util.feature_flag import Flag, is_feature_enabled-from backend.util.retry import func_retryfrom backend.util.settings import Settingsfrom backend.util.truncate import truncate@@ -70,6 +69,13 @@ class NodeRelation(TypedDict):sink_block_name: NotRequired[str] # Optional, only set if block exists+class ActivityStatusResponse(TypedDict):+ """Type definition for structured activity status response."""++ activity_status: str+ correctness_score: float++def _truncate_uuid(uuid_str: str) -> str:"""Truncate UUID to first segment to reduce payload size."""if not uuid_str:@@ -85,9 +91,9 @@ async def generate_activity_status_for_execution(db_client: "DatabaseManagerAsyncClient",user_id: str,execution_status: ExecutionStatus | None = None,-) -> str | None:+) -> ActivityStatusResponse | None:"""- Generate an AI-based activity status summary for a graph execution.+ Generate an AI-based activity status summary and correctness assessment for a graph execution.This function handles all the data collection and AI generation logic,keeping the manager integration simple.@@ -102,7 +108,8 @@ async def generate_activity_status_for_execution(execution_status: The overall execution status (COMPLETED, FAILED, TERMINATED)Returns:- AI-generated activity status string, or None if feature is disabled+ AI-generated activity status response with activity_status and correctness_status,+ or None if feature is disabled"""# Check LaunchDarkly feature flag for AI activity status generation with full context supportif not await is_feature_enabled(Flag.AI_ACTIVITY_STATUS, user_id):@@ -141,16 +148,27 @@ async def generate_activity_status_for_execution(execution_status,)- # Prepare prompt for AI+ # Prepare prompt for AI with structured output requirementsprompt = [{"role": "system","content": (- "You are an AI assistant summarizing what you just did for a user in simple, friendly language. "- "Write from the user's perspective about what they accomplished, NOT about technical execution details. "- "Focus on the ACTUAL TASK the user wanted done, not the internal workflow steps. "- "Avoid technical terms like 'workflow', 'execution', 'components', 'nodes', 'processing', etc. "- "Keep it to 3 sentences maximum. Be conversational and human-friendly.\n\n"+ "You are an AI assistant analyzing what an agent execution accomplished and whether it worked correctly. "+ "You need to provide both a user-friendly summary AND a correctness assessment.\n\n"+ "FOR THE ACTIVITY STATUS:\n"+ "- Write from the user's perspective about what they accomplished, NOT about technical execution details\n"+ "- Focus on the ACTUAL TASK the user wanted done, not the internal workflow steps\n"+ "- Avoid technical terms like 'workflow', 'execution', 'components', 'nodes', 'processing', etc.\n"+ "- Keep it to 3 sentences maximum. Be conversational and human-friendly\n\n"+ "FOR THE CORRECTNESS SCORE:\n"+ "- Provide a score from 0.0 to 1.0 indicating how well the execution achieved its intended purpose\n"+ "- Use this scoring guide:\n"+ " 0.0-0.2: Failure - The result clearly did not meet the task requirements\n"+ " 0.2-0.4: Poor - Major issues; only small parts of the goal were achieved\n"+ " 0.4-0.6: Partial Success - Some objectives met, but with noticeable gaps or inaccuracies\n"+ " 0.6-0.8: Mostly Successful - Largely achieved the intended outcome, with minor flaws\n"+ " 0.8-1.0: Success - Fully met or exceeded the task requirements\n"+ "- Base the score on actual outputs produced, not just technical completion\n\n""UNDERSTAND THE INTENDED PURPOSE:\n""- FIRST: Read the graph description carefully to understand what the user wanted to accomplish\n""- The graph name and description tell you the main goal/intention of this automation\n"@@ -186,7 +204,7 @@ async def generate_activity_status_for_execution("role": "user","content": (f"A user ran '{graph_name}' to accomplish something. Based on this execution data, "- f"write what they achieved in simple, user-friendly terms:\n\n"+ f"provide both an activity summary and correctness assessment:\n\n"f"{json.dumps(execution_data, indent=2)}\n\n""ANALYSIS CHECKLIST:\n""1. READ graph_info.description FIRST - this tells you what the user intended to accomplish\n"@@ -203,13 +221,20 @@ async def generate_activity_status_for_execution("- If description mentions 'content generation' → was content actually generated?\n""- If description mentions 'social media posting' → were posts actually made?\n""- Match the outputs to the stated intention, not just technical completion\n\n"- "Write 1-3 sentences about what the user accomplished, such as:\n"+ "PROVIDE:\n"+ "activity_status: 1-3 sentences about what the user accomplished, such as:\n""- 'I analyzed your resume and provided detailed feedback for the IT industry.'\n""- 'I couldn't complete the task because critical steps failed to produce any results.'\n""- 'I failed to generate the content you requested due to missing API access.'\n""- 'I extracted key information from your documents and organized it into a summary.'\n""- 'The task failed because the blog post creation step didn't produce any output.'\n\n"- "BE CRITICAL: If the graph's intended purpose (from description) wasn't achieved, report this as a failure even if status is 'completed'."+ "correctness_score: A float score from 0.0 to 1.0 based on how well the intended purpose was achieved:\n"+ "- 0.0-0.2: Failure (didn't meet requirements)\n"+ "- 0.2-0.4: Poor (major issues, minimal achievement)\n"+ "- 0.4-0.6: Partial Success (some objectives met with gaps)\n"+ "- 0.6-0.8: Mostly Successful (largely achieved with minor flaws)\n"+ "- 0.8-1.0: Success (fully met or exceeded requirements)\n\n"+ "BE CRITICAL: If the graph's intended purpose (from description) wasn't achieved, use a low score (0.0-0.4) even if status is 'completed'."),},]@@ -227,14 +252,58 @@ async def generate_activity_status_for_execution(title="System OpenAI",)- # Make LLM call using current event loop- activity_status = await _call_llm_direct(credentials, prompt)+ # Define expected response format+ expected_format = {+ "activity_status": "A user-friendly 1-3 sentence summary of what was accomplished",+ "correctness_score": "Float score from 0.0 to 1.0 indicating how well the execution achieved its intended purpose",+ }++ # Use existing AIStructuredResponseGeneratorBlock for structured LLM call+ structured_block = AIStructuredResponseGeneratorBlock()++ # Convert credentials to the format expected by AIStructuredResponseGeneratorBlock+ credentials_input = {+ "provider": credentials.provider,+ "id": credentials.id,+ "type": credentials.type,+ "title": credentials.title,+ }++ structured_input = AIStructuredResponseGeneratorBlock.Input(+ prompt=prompt[1]["content"], # User prompt content+ sys_prompt=prompt[0]["content"], # System prompt content+ expected_format=expected_format,+ model=LlmModel.GPT4O_MINI,+ credentials=credentials_input, # type: ignore+ max_tokens=150,+ retry=3,+ )++ # Execute the structured LLM call+ async for output_name, output_data in structured_block.run(+ structured_input, credentials=credentials+ ):+ if output_name == "response":+ response = output_data+ break+ else:+ raise RuntimeError("Failed to get response from structured LLM call")++ # Create typed response with validation+ correctness_score = float(response["correctness_score"])+ # Clamp score to valid range+ correctness_score = max(0.0, min(1.0, correctness_score))++ activity_response: ActivityStatusResponse = {+ "activity_status": response["activity_status"],+ "correctness_score": correctness_score,+ }logger.debug(- f"Generated activity status for {graph_exec_id}: {activity_status}"+ f"Generated activity status for {graph_exec_id}: {activity_response}")- return activity_status+ return activity_responseexcept Exception as e:logger.error(@@ -448,23 +517,3 @@ def _build_execution_summary(),},}---@func_retry-async def _call_llm_direct(- credentials: APIKeyCredentials, prompt: list[dict[str, str]]-) -> str:- """Make direct LLM call."""-- response = await llm_call(- credentials=credentials,- llm_model=LlmModel.GPT4O_MINI,- prompt=prompt,- max_tokens=150,- compress_prompt_to_fit=True,- )-- if response and response.response:- return response.response.strip()- else:- return "Unable to generate activity summary"diff --git a/autogpt_platform/backend/backend/executor/activity_status_generator_test.py b/autogpt_platform/backend/backend/executor/activity_status_generator_test.pyindex 206be21d02..c3ce0b6bf0 100644--- a/autogpt_platform/backend/backend/executor/activity_status_generator_test.py+++ b/autogpt_platform/backend/backend/executor/activity_status_generator_test.py@@ -7,12 +7,11 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest-from backend.blocks.llm import LLMResponse+from backend.blocks.llm import LlmModel, LLMResponsefrom backend.data.execution import ExecutionStatus, NodeExecutionResultfrom backend.data.model import GraphExecutionStatsfrom backend.executor.activity_status_generator import (_build_execution_summary,- _call_llm_direct,generate_activity_status_for_execution,)@@ -373,25 +372,24 @@ class TestLLMCall:"""Tests for LLM calling functionality."""@pytest.mark.asyncio- async def test_call_llm_direct_success(self):- """Test successful LLM call."""+ async def test_structured_llm_call_success(self):+ """Test successful structured LLM call."""from pydantic import SecretStr+ from backend.blocks.llm import AIStructuredResponseGeneratorBlockfrom backend.data.model import APIKeyCredentials- mock_response = LLMResponse(- raw_response={},- prompt=[],- response="Agent successfully processed user input and generated response.",- tool_calls=None,- prompt_tokens=50,- completion_tokens=20,- )-- with patch(- "backend.executor.activity_status_generator.llm_call"- ) as mock_llm_call:- mock_llm_call.return_value = mock_response+ with patch("backend.blocks.llm.llm_call") as mock_llm_call, patch(+ "backend.blocks.llm.secrets.token_hex", return_value="test123"+ ):+ mock_llm_call.return_value = LLMResponse(+ raw_response={},+ prompt=[],+ response='<json_output id="test123">{"activity_status": "Test completed successfully", "correctness_score": 0.9}</json_output>',+ tool_calls=None,+ prompt_tokens=50,+ completion_tokens=20,+ )credentials = APIKeyCredentials(id="test",@@ -401,26 +399,61 @@ class TestLLMCall:)prompt = [{"role": "user", "content": "Test prompt"}]+ expected_format = {+ "activity_status": "User-friendly summary",+ "correctness_score": "Float score from 0.0 to 1.0",+ }++ # Create structured block and input+ structured_block = AIStructuredResponseGeneratorBlock()+ credentials_input = {+ "provider": credentials.provider,+ "id": credentials.id,+ "type": credentials.type,+ "title": credentials.title,+ }++ structured_input = AIStructuredResponseGeneratorBlock.Input(+ prompt=prompt[0]["content"],+ expected_format=expected_format,+ model=LlmModel.GPT4O_MINI,+ credentials=credentials_input, # type: ignore+ )- result = await _call_llm_direct(credentials, prompt)+ # Execute the structured LLM call+ result = None+ async for output_name, output_data in structured_block.run(+ structured_input, credentials=credentials+ ):+ if output_name == "response":+ result = output_data+ break- assert (- result- == "Agent successfully processed user input and generated response."- )- mock_llm_call.assert_called_once()+ assert result is not None+ assert result["activity_status"] == "Test completed successfully"+ assert result["correctness_score"] == 0.9+ mock_llm_call.assert_called()@pytest.mark.asyncio- async def test_call_llm_direct_no_response(self):- """Test LLM call with no response."""+ async def test_structured_llm_call_validation_error(self):+ """Test structured LLM call with validation error."""from pydantic import SecretStr+ from backend.blocks.llm import AIStructuredResponseGeneratorBlockfrom backend.data.model import APIKeyCredentials- with patch(- "backend.executor.activity_status_generator.llm_call"- ) as mock_llm_call:- mock_llm_call.return_value = None+ with patch("backend.blocks.llm.llm_call") as mock_llm_call, patch(+ "backend.blocks.llm.secrets.token_hex", return_value="test123"+ ):+ # Return invalid JSON that will fail validation (missing required field)+ mock_llm_call.return_value = LLMResponse(+ raw_response={},+ prompt=[],+ response='<json_output id="test123">{"activity_status": "Test completed successfully"}</json_output>',+ tool_calls=None,+ prompt_tokens=50,+ completion_tokens=20,+ )credentials = APIKeyCredentials(id="test",@@ -430,10 +463,36 @@ class TestLLMCall:)prompt = [{"role": "user", "content": "Test prompt"}]+ expected_format = {+ "activity_status": "User-friendly summary",+ "correctness_score": "Float score from 0.0 to 1.0",+ }++ # Create structured block and input+ structured_block = AIStructuredResponseGeneratorBlock()+ credentials_input = {+ "provider": credentials.provider,+ "id": credentials.id,+ "type": credentials.type,+ "title": credentials.title,+ }++ structured_input = AIStructuredResponseGeneratorBlock.Input(+ prompt=prompt[0]["content"],+ expected_format=expected_format,+ model=LlmModel.GPT4O_MINI,+ credentials=credentials_input, # type: ignore+ retry=1, # Use fewer retries for faster test+ )- result = await _call_llm_direct(credentials, prompt)-- assert result == "Unable to generate activity summary"+ with pytest.raises(+ Exception+ ): # AIStructuredResponseGeneratorBlock may raise different exceptions+ async for output_name, output_data in structured_block.run(+ structured_input, credentials=credentials+ ):+ if output_name == "response":+ breakclass TestGenerateActivityStatusForExecution:@@ -461,17 +520,25 @@ class TestGenerateActivityStatusForExecution:) as mock_get_block, patch("backend.executor.activity_status_generator.Settings") as mock_settings, patch(- "backend.executor.activity_status_generator._call_llm_direct"- ) as mock_llm, patch(+ "backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"+ ) as mock_structured_block, patch("backend.executor.activity_status_generator.is_feature_enabled",return_value=True,):mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)mock_settings.return_value.secrets.openai_internal_api_key = "test_key"- mock_llm.return_value = (- "I analyzed your data and provided the requested insights."- )++ # Mock the structured block to return our expected response+ mock_instance = mock_structured_block.return_value++ async def mock_run(*args, **kwargs):+ yield "response", {+ "activity_status": "I analyzed your data and provided the requested insights.",+ "correctness_score": 0.85,+ }++ mock_instance.run = mock_runresult = await generate_activity_status_for_execution(graph_exec_id="test_exec",@@ -482,11 +549,16 @@ class TestGenerateActivityStatusForExecution:user_id="test_user",)- assert result == "I analyzed your data and provided the requested insights."+ assert result is not None+ assert (+ result["activity_status"]+ == "I analyzed your data and provided the requested insights."+ )+ assert result["correctness_score"] == 0.85mock_db_client.get_node_executions.assert_called_once()mock_db_client.get_graph_metadata.assert_called_once()mock_db_client.get_graph.assert_called_once()- mock_llm.assert_called_once()+ mock_structured_block.assert_called_once()@pytest.mark.asyncioasync def test_generate_status_feature_disabled(self, mock_execution_stats):@@ -574,15 +646,25 @@ class TestGenerateActivityStatusForExecution:) as mock_get_block, patch("backend.executor.activity_status_generator.Settings") as mock_settings, patch(- "backend.executor.activity_status_generator._call_llm_direct"- ) as mock_llm, patch(+ "backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"+ ) as mock_structured_block, patch("backend.executor.activity_status_generator.is_feature_enabled",return_value=True,):mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)mock_settings.return_value.secrets.openai_internal_api_key = "test_key"- mock_llm.return_value = "Agent completed execution."++ # Mock the structured block to return our expected response+ mock_instance = mock_structured_block.return_value++ async def mock_run(*args, **kwargs):+ yield "response", {+ "activity_status": "Agent completed execution.",+ "correctness_score": 0.8,+ }++ mock_instance.run = mock_runresult = await generate_activity_status_for_execution(graph_exec_id="test_exec",@@ -593,10 +675,11 @@ class TestGenerateActivityStatusForExecution:user_id="test_user",)- assert result == "Agent completed execution."- # Should use fallback graph name in prompt- call_args = mock_llm.call_args[0][1] # prompt argument- assert "Graph test_graph" in call_args[1]["content"]+ assert result is not None+ assert result["activity_status"] == "Agent completed execution."+ assert result["correctness_score"] == 0.8+ # The structured block should have been instantiated+ assert mock_structured_block.calledclass TestIntegration:@@ -626,8 +709,8 @@ class TestIntegration:) as mock_get_block, patch("backend.executor.activity_status_generator.Settings") as mock_settings, patch(- "backend.executor.activity_status_generator.llm_call"- ) as mock_llm_call, patch(+ "backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"+ ) as mock_structured_block, patch("backend.executor.activity_status_generator.is_feature_enabled",return_value=True,):@@ -635,15 +718,16 @@ class TestIntegration:mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)mock_settings.return_value.secrets.openai_internal_api_key = "test_key"- mock_response = LLMResponse(- raw_response={},- prompt=[],- response=expected_activity,- tool_calls=None,- prompt_tokens=100,- completion_tokens=30,- )- mock_llm_call.return_value = mock_response+ # Mock the structured block to return our expected response+ mock_instance = mock_structured_block.return_value++ async def mock_run(*args, **kwargs):+ yield "response", {+ "activity_status": expected_activity,+ "correctness_score": 0.3, # Low score since there was a failure+ }++ mock_instance.run = mock_runresult = await generate_activity_status_for_execution(graph_exec_id="test_exec",@@ -654,24 +738,14 @@ class TestIntegration:user_id="test_user",)- assert result == expected_activity-- # Verify the correct data was passed to LLM- llm_call_args = mock_llm_call.call_args- prompt = llm_call_args[1]["prompt"]-- # Check system prompt- assert prompt[0]["role"] == "system"- assert "user's perspective" in prompt[0]["content"]-- # Check user prompt contains expected data- user_content = prompt[1]["content"]- assert "Test Integration Agent" in user_content- assert "user-friendly terms" in user_content.lower()+ assert result is not None+ assert result["activity_status"] == expected_activity+ assert result["correctness_score"] == 0.3- # Verify that execution data is present in the prompt- assert "{" in user_content # Should contain JSON data- assert "overall_status" in user_content+ # Verify the structured block was called+ assert mock_structured_block.called+ # The structured block should have been instantiated+ mock_structured_block.assert_called_once()@pytest.mark.asyncioasync def test_manager_integration_with_disabled_feature(diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.pyindex 5b433d4081..3863984039 100644--- a/autogpt_platform/backend/backend/executor/manager.py+++ b/autogpt_platform/backend/backend/executor/manager.py@@ -696,7 +696,7 @@ class ExecutionProcessor:exec_meta.status = status# Activity status handling- activity_status = asyncio.run_coroutine_threadsafe(+ activity_response = asyncio.run_coroutine_threadsafe(generate_activity_status_for_execution(graph_exec_id=graph_exec.graph_exec_id,graph_id=graph_exec.graph_id,@@ -708,18 +708,21 @@ class ExecutionProcessor:),self.node_execution_loop,).result(timeout=60.0)- if activity_status is not None:- exec_stats.activity_status = activity_status- log_metadata.info(f"Generated activity status: {activity_status}")+ if activity_response is not None:+ exec_stats.activity_status = activity_response["activity_status"]+ exec_stats.correctness_score = activity_response["correctness_score"]+ log_metadata.info(+ f"Generated activity status: {activity_response['activity_status']} "+ f"(correctness: {activity_response['correctness_score']:.2f})"+ )else:log_metadata.debug(- "Activity status generation disabled, not setting field"+ "Activity status generation disabled, not setting fields")-+ finally:# Communication handlingself._handle_agent_run_notif(db_client, graph_exec, exec_stats)- finally:update_graph_execution_state(db_client=db_client,graph_exec_id=graph_exec.graph_exec_id,diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.jsonindex 43ee6b4fc2..9ea38ba7bb 100644--- a/autogpt_platform/frontend/src/app/api/openapi.json+++ b/autogpt_platform/frontend/src/app/api/openapi.json@@ -7821,6 +7821,11 @@"anyOf": [{ "type": "string" }, { "type": "null" }],"title": "Activity Status","description": "AI-generated summary of what the agent did"+ },+ "correctness_score": {+ "anyOf": [{ "type": "number" }, { "type": "null" }],+ "title": "Correctness Score",+ "description": "AI-generated score (0.0-1.0) indicating how well the execution achieved its intended purpose"}},"additionalProperties": true,