Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): Add correctness score to execution activity generation (#11325)

Zamil Majdy6037f8050297
diff --git a/autogpt_platform/backend/backend/data/execution.py b/autogpt_platform/backend/backend/data/execution.py
index 922672acd0..8025fc5d81 100644
--- a/autogpt_platform/backend/backend/data/execution.py
+++ b/autogpt_platform/backend/backend/data/execution.py
@@ -175,6 +175,10 @@ class GraphExecutionMeta(BaseDbModel):
default=None,
description="AI-generated summary of what the agent did",
)
+ correctness_score: float | None = Field(
+ default=None,
+ description="AI-generated score (0.0-1.0) indicating how well the execution achieved its intended purpose",
+ )
def to_db(self) -> GraphExecutionStats:
return GraphExecutionStats(
@@ -187,6 +191,7 @@ class GraphExecutionMeta(BaseDbModel):
node_error_count=self.node_error_count,
error=self.error,
activity_status=self.activity_status,
+ correctness_score=self.correctness_score,
)
stats: Stats | None
@@ -244,6 +249,7 @@ class GraphExecutionMeta(BaseDbModel):
else stats.error
),
activity_status=stats.activity_status,
+ correctness_score=stats.correctness_score,
)
if stats
else None
diff --git a/autogpt_platform/backend/backend/data/model.py b/autogpt_platform/backend/backend/data/model.py
index ac3c85461d..1c32a9f444 100644
--- a/autogpt_platform/backend/backend/data/model.py
+++ b/autogpt_platform/backend/backend/data/model.py
@@ -833,6 +833,10 @@ class GraphExecutionStats(BaseModel):
activity_status: Optional[str] = Field(
default=None, description="AI-generated summary of what the agent did"
)
+ correctness_score: Optional[float] = Field(
+ default=None,
+ description="AI-generated score (0.0-1.0) indicating how well the execution achieved its intended purpose",
+ )
class UserExecutionSummaryStats(BaseModel):
diff --git a/autogpt_platform/backend/backend/executor/activity_status_generator.py b/autogpt_platform/backend/backend/executor/activity_status_generator.py
index b9d1ed7bd5..faa11a4410 100644
--- a/autogpt_platform/backend/backend/executor/activity_status_generator.py
+++ b/autogpt_platform/backend/backend/executor/activity_status_generator.py
@@ -13,12 +13,11 @@ except ImportError:
from pydantic import SecretStr
-from backend.blocks.llm import LlmModel, llm_call
+from backend.blocks.llm import AIStructuredResponseGeneratorBlock, LlmModel
from backend.data.block import get_block
from backend.data.execution import ExecutionStatus, NodeExecutionResult
from backend.data.model import APIKeyCredentials, GraphExecutionStats
from backend.util.feature_flag import Flag, is_feature_enabled
-from backend.util.retry import func_retry
from backend.util.settings import Settings
from backend.util.truncate import truncate
@@ -70,6 +69,13 @@ class NodeRelation(TypedDict):
sink_block_name: NotRequired[str] # Optional, only set if block exists
+class ActivityStatusResponse(TypedDict):
+ """Type definition for structured activity status response."""
+
+ activity_status: str
+ correctness_score: float
+
+
def _truncate_uuid(uuid_str: str) -> str:
"""Truncate UUID to first segment to reduce payload size."""
if not uuid_str:
@@ -85,9 +91,9 @@ async def generate_activity_status_for_execution(
db_client: "DatabaseManagerAsyncClient",
user_id: str,
execution_status: ExecutionStatus | None = None,
-) -> str | None:
+) -> ActivityStatusResponse | None:
"""
- Generate an AI-based activity status summary for a graph execution.
+ Generate an AI-based activity status summary and correctness assessment for a graph execution.
This function handles all the data collection and AI generation logic,
keeping the manager integration simple.
@@ -102,7 +108,8 @@ async def generate_activity_status_for_execution(
execution_status: The overall execution status (COMPLETED, FAILED, TERMINATED)
Returns:
- AI-generated activity status string, or None if feature is disabled
+ AI-generated activity status response with activity_status and correctness_status,
+ or None if feature is disabled
"""
# Check LaunchDarkly feature flag for AI activity status generation with full context support
if not await is_feature_enabled(Flag.AI_ACTIVITY_STATUS, user_id):
@@ -141,16 +148,27 @@ async def generate_activity_status_for_execution(
execution_status,
)
- # Prepare prompt for AI
+ # Prepare prompt for AI with structured output requirements
prompt = [
{
"role": "system",
"content": (
- "You are an AI assistant summarizing what you just did for a user in simple, friendly language. "
- "Write from the user's perspective about what they accomplished, NOT about technical execution details. "
- "Focus on the ACTUAL TASK the user wanted done, not the internal workflow steps. "
- "Avoid technical terms like 'workflow', 'execution', 'components', 'nodes', 'processing', etc. "
- "Keep it to 3 sentences maximum. Be conversational and human-friendly.\n\n"
+ "You are an AI assistant analyzing what an agent execution accomplished and whether it worked correctly. "
+ "You need to provide both a user-friendly summary AND a correctness assessment.\n\n"
+ "FOR THE ACTIVITY STATUS:\n"
+ "- Write from the user's perspective about what they accomplished, NOT about technical execution details\n"
+ "- Focus on the ACTUAL TASK the user wanted done, not the internal workflow steps\n"
+ "- Avoid technical terms like 'workflow', 'execution', 'components', 'nodes', 'processing', etc.\n"
+ "- Keep it to 3 sentences maximum. Be conversational and human-friendly\n\n"
+ "FOR THE CORRECTNESS SCORE:\n"
+ "- Provide a score from 0.0 to 1.0 indicating how well the execution achieved its intended purpose\n"
+ "- Use this scoring guide:\n"
+ " 0.0-0.2: Failure - The result clearly did not meet the task requirements\n"
+ " 0.2-0.4: Poor - Major issues; only small parts of the goal were achieved\n"
+ " 0.4-0.6: Partial Success - Some objectives met, but with noticeable gaps or inaccuracies\n"
+ " 0.6-0.8: Mostly Successful - Largely achieved the intended outcome, with minor flaws\n"
+ " 0.8-1.0: Success - Fully met or exceeded the task requirements\n"
+ "- Base the score on actual outputs produced, not just technical completion\n\n"
"UNDERSTAND THE INTENDED PURPOSE:\n"
"- FIRST: Read the graph description carefully to understand what the user wanted to accomplish\n"
"- The graph name and description tell you the main goal/intention of this automation\n"
@@ -186,7 +204,7 @@ async def generate_activity_status_for_execution(
"role": "user",
"content": (
f"A user ran '{graph_name}' to accomplish something. Based on this execution data, "
- f"write what they achieved in simple, user-friendly terms:\n\n"
+ f"provide both an activity summary and correctness assessment:\n\n"
f"{json.dumps(execution_data, indent=2)}\n\n"
"ANALYSIS CHECKLIST:\n"
"1. READ graph_info.description FIRST - this tells you what the user intended to accomplish\n"
@@ -203,13 +221,20 @@ async def generate_activity_status_for_execution(
"- If description mentions 'content generation' → was content actually generated?\n"
"- If description mentions 'social media posting' → were posts actually made?\n"
"- Match the outputs to the stated intention, not just technical completion\n\n"
- "Write 1-3 sentences about what the user accomplished, such as:\n"
+ "PROVIDE:\n"
+ "activity_status: 1-3 sentences about what the user accomplished, such as:\n"
"- 'I analyzed your resume and provided detailed feedback for the IT industry.'\n"
"- 'I couldn't complete the task because critical steps failed to produce any results.'\n"
"- 'I failed to generate the content you requested due to missing API access.'\n"
"- 'I extracted key information from your documents and organized it into a summary.'\n"
"- 'The task failed because the blog post creation step didn't produce any output.'\n\n"
- "BE CRITICAL: If the graph's intended purpose (from description) wasn't achieved, report this as a failure even if status is 'completed'."
+ "correctness_score: A float score from 0.0 to 1.0 based on how well the intended purpose was achieved:\n"
+ "- 0.0-0.2: Failure (didn't meet requirements)\n"
+ "- 0.2-0.4: Poor (major issues, minimal achievement)\n"
+ "- 0.4-0.6: Partial Success (some objectives met with gaps)\n"
+ "- 0.6-0.8: Mostly Successful (largely achieved with minor flaws)\n"
+ "- 0.8-1.0: Success (fully met or exceeded requirements)\n\n"
+ "BE CRITICAL: If the graph's intended purpose (from description) wasn't achieved, use a low score (0.0-0.4) even if status is 'completed'."
),
},
]
@@ -227,14 +252,58 @@ async def generate_activity_status_for_execution(
title="System OpenAI",
)
- # Make LLM call using current event loop
- activity_status = await _call_llm_direct(credentials, prompt)
+ # Define expected response format
+ expected_format = {
+ "activity_status": "A user-friendly 1-3 sentence summary of what was accomplished",
+ "correctness_score": "Float score from 0.0 to 1.0 indicating how well the execution achieved its intended purpose",
+ }
+
+ # Use existing AIStructuredResponseGeneratorBlock for structured LLM call
+ structured_block = AIStructuredResponseGeneratorBlock()
+
+ # Convert credentials to the format expected by AIStructuredResponseGeneratorBlock
+ credentials_input = {
+ "provider": credentials.provider,
+ "id": credentials.id,
+ "type": credentials.type,
+ "title": credentials.title,
+ }
+
+ structured_input = AIStructuredResponseGeneratorBlock.Input(
+ prompt=prompt[1]["content"], # User prompt content
+ sys_prompt=prompt[0]["content"], # System prompt content
+ expected_format=expected_format,
+ model=LlmModel.GPT4O_MINI,
+ credentials=credentials_input, # type: ignore
+ max_tokens=150,
+ retry=3,
+ )
+
+ # Execute the structured LLM call
+ async for output_name, output_data in structured_block.run(
+ structured_input, credentials=credentials
+ ):
+ if output_name == "response":
+ response = output_data
+ break
+ else:
+ raise RuntimeError("Failed to get response from structured LLM call")
+
+ # Create typed response with validation
+ correctness_score = float(response["correctness_score"])
+ # Clamp score to valid range
+ correctness_score = max(0.0, min(1.0, correctness_score))
+
+ activity_response: ActivityStatusResponse = {
+ "activity_status": response["activity_status"],
+ "correctness_score": correctness_score,
+ }
logger.debug(
- f"Generated activity status for {graph_exec_id}: {activity_status}"
+ f"Generated activity status for {graph_exec_id}: {activity_response}"
)
- return activity_status
+ return activity_response
except Exception as e:
logger.error(
@@ -448,23 +517,3 @@ def _build_execution_summary(
),
},
}
-
-
-@func_retry
-async def _call_llm_direct(
- credentials: APIKeyCredentials, prompt: list[dict[str, str]]
-) -> str:
- """Make direct LLM call."""
-
- response = await llm_call(
- credentials=credentials,
- llm_model=LlmModel.GPT4O_MINI,
- prompt=prompt,
- max_tokens=150,
- compress_prompt_to_fit=True,
- )
-
- if response and response.response:
- return response.response.strip()
- else:
- return "Unable to generate activity summary"
diff --git a/autogpt_platform/backend/backend/executor/activity_status_generator_test.py b/autogpt_platform/backend/backend/executor/activity_status_generator_test.py
index 206be21d02..c3ce0b6bf0 100644
--- a/autogpt_platform/backend/backend/executor/activity_status_generator_test.py
+++ b/autogpt_platform/backend/backend/executor/activity_status_generator_test.py
@@ -7,12 +7,11 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
-from backend.blocks.llm import LLMResponse
+from backend.blocks.llm import LlmModel, LLMResponse
from backend.data.execution import ExecutionStatus, NodeExecutionResult
from backend.data.model import GraphExecutionStats
from backend.executor.activity_status_generator import (
_build_execution_summary,
- _call_llm_direct,
generate_activity_status_for_execution,
)
@@ -373,25 +372,24 @@ class TestLLMCall:
"""Tests for LLM calling functionality."""
@pytest.mark.asyncio
- async def test_call_llm_direct_success(self):
- """Test successful LLM call."""
+ async def test_structured_llm_call_success(self):
+ """Test successful structured LLM call."""
from pydantic import SecretStr
+ from backend.blocks.llm import AIStructuredResponseGeneratorBlock
from backend.data.model import APIKeyCredentials
- mock_response = LLMResponse(
- raw_response={},
- prompt=[],
- response="Agent successfully processed user input and generated response.",
- tool_calls=None,
- prompt_tokens=50,
- completion_tokens=20,
- )
-
- with patch(
- "backend.executor.activity_status_generator.llm_call"
- ) as mock_llm_call:
- mock_llm_call.return_value = mock_response
+ with patch("backend.blocks.llm.llm_call") as mock_llm_call, patch(
+ "backend.blocks.llm.secrets.token_hex", return_value="test123"
+ ):
+ mock_llm_call.return_value = LLMResponse(
+ raw_response={},
+ prompt=[],
+ response='<json_output id="test123">{"activity_status": "Test completed successfully", "correctness_score": 0.9}</json_output>',
+ tool_calls=None,
+ prompt_tokens=50,
+ completion_tokens=20,
+ )
credentials = APIKeyCredentials(
id="test",
@@ -401,26 +399,61 @@ class TestLLMCall:
)
prompt = [{"role": "user", "content": "Test prompt"}]
+ expected_format = {
+ "activity_status": "User-friendly summary",
+ "correctness_score": "Float score from 0.0 to 1.0",
+ }
+
+ # Create structured block and input
+ structured_block = AIStructuredResponseGeneratorBlock()
+ credentials_input = {
+ "provider": credentials.provider,
+ "id": credentials.id,
+ "type": credentials.type,
+ "title": credentials.title,
+ }
+
+ structured_input = AIStructuredResponseGeneratorBlock.Input(
+ prompt=prompt[0]["content"],
+ expected_format=expected_format,
+ model=LlmModel.GPT4O_MINI,
+ credentials=credentials_input, # type: ignore
+ )
- result = await _call_llm_direct(credentials, prompt)
+ # Execute the structured LLM call
+ result = None
+ async for output_name, output_data in structured_block.run(
+ structured_input, credentials=credentials
+ ):
+ if output_name == "response":
+ result = output_data
+ break
- assert (
- result
- == "Agent successfully processed user input and generated response."
- )
- mock_llm_call.assert_called_once()
+ assert result is not None
+ assert result["activity_status"] == "Test completed successfully"
+ assert result["correctness_score"] == 0.9
+ mock_llm_call.assert_called()
@pytest.mark.asyncio
- async def test_call_llm_direct_no_response(self):
- """Test LLM call with no response."""
+ async def test_structured_llm_call_validation_error(self):
+ """Test structured LLM call with validation error."""
from pydantic import SecretStr
+ from backend.blocks.llm import AIStructuredResponseGeneratorBlock
from backend.data.model import APIKeyCredentials
- with patch(
- "backend.executor.activity_status_generator.llm_call"
- ) as mock_llm_call:
- mock_llm_call.return_value = None
+ with patch("backend.blocks.llm.llm_call") as mock_llm_call, patch(
+ "backend.blocks.llm.secrets.token_hex", return_value="test123"
+ ):
+ # Return invalid JSON that will fail validation (missing required field)
+ mock_llm_call.return_value = LLMResponse(
+ raw_response={},
+ prompt=[],
+ response='<json_output id="test123">{"activity_status": "Test completed successfully"}</json_output>',
+ tool_calls=None,
+ prompt_tokens=50,
+ completion_tokens=20,
+ )
credentials = APIKeyCredentials(
id="test",
@@ -430,10 +463,36 @@ class TestLLMCall:
)
prompt = [{"role": "user", "content": "Test prompt"}]
+ expected_format = {
+ "activity_status": "User-friendly summary",
+ "correctness_score": "Float score from 0.0 to 1.0",
+ }
+
+ # Create structured block and input
+ structured_block = AIStructuredResponseGeneratorBlock()
+ credentials_input = {
+ "provider": credentials.provider,
+ "id": credentials.id,
+ "type": credentials.type,
+ "title": credentials.title,
+ }
+
+ structured_input = AIStructuredResponseGeneratorBlock.Input(
+ prompt=prompt[0]["content"],
+ expected_format=expected_format,
+ model=LlmModel.GPT4O_MINI,
+ credentials=credentials_input, # type: ignore
+ retry=1, # Use fewer retries for faster test
+ )
- result = await _call_llm_direct(credentials, prompt)
-
- assert result == "Unable to generate activity summary"
+ with pytest.raises(
+ Exception
+ ): # AIStructuredResponseGeneratorBlock may raise different exceptions
+ async for output_name, output_data in structured_block.run(
+ structured_input, credentials=credentials
+ ):
+ if output_name == "response":
+ break
class TestGenerateActivityStatusForExecution:
@@ -461,17 +520,25 @@ class TestGenerateActivityStatusForExecution:
) as mock_get_block, patch(
"backend.executor.activity_status_generator.Settings"
) as mock_settings, patch(
- "backend.executor.activity_status_generator._call_llm_direct"
- ) as mock_llm, patch(
+ "backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"
+ ) as mock_structured_block, patch(
"backend.executor.activity_status_generator.is_feature_enabled",
return_value=True,
):
mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)
mock_settings.return_value.secrets.openai_internal_api_key = "test_key"
- mock_llm.return_value = (
- "I analyzed your data and provided the requested insights."
- )
+
+ # Mock the structured block to return our expected response
+ mock_instance = mock_structured_block.return_value
+
+ async def mock_run(*args, **kwargs):
+ yield "response", {
+ "activity_status": "I analyzed your data and provided the requested insights.",
+ "correctness_score": 0.85,
+ }
+
+ mock_instance.run = mock_run
result = await generate_activity_status_for_execution(
graph_exec_id="test_exec",
@@ -482,11 +549,16 @@ class TestGenerateActivityStatusForExecution:
user_id="test_user",
)
- assert result == "I analyzed your data and provided the requested insights."
+ assert result is not None
+ assert (
+ result["activity_status"]
+ == "I analyzed your data and provided the requested insights."
+ )
+ assert result["correctness_score"] == 0.85
mock_db_client.get_node_executions.assert_called_once()
mock_db_client.get_graph_metadata.assert_called_once()
mock_db_client.get_graph.assert_called_once()
- mock_llm.assert_called_once()
+ mock_structured_block.assert_called_once()
@pytest.mark.asyncio
async def test_generate_status_feature_disabled(self, mock_execution_stats):
@@ -574,15 +646,25 @@ class TestGenerateActivityStatusForExecution:
) as mock_get_block, patch(
"backend.executor.activity_status_generator.Settings"
) as mock_settings, patch(
- "backend.executor.activity_status_generator._call_llm_direct"
- ) as mock_llm, patch(
+ "backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"
+ ) as mock_structured_block, patch(
"backend.executor.activity_status_generator.is_feature_enabled",
return_value=True,
):
mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)
mock_settings.return_value.secrets.openai_internal_api_key = "test_key"
- mock_llm.return_value = "Agent completed execution."
+
+ # Mock the structured block to return our expected response
+ mock_instance = mock_structured_block.return_value
+
+ async def mock_run(*args, **kwargs):
+ yield "response", {
+ "activity_status": "Agent completed execution.",
+ "correctness_score": 0.8,
+ }
+
+ mock_instance.run = mock_run
result = await generate_activity_status_for_execution(
graph_exec_id="test_exec",
@@ -593,10 +675,11 @@ class TestGenerateActivityStatusForExecution:
user_id="test_user",
)
- assert result == "Agent completed execution."
- # Should use fallback graph name in prompt
- call_args = mock_llm.call_args[0][1] # prompt argument
- assert "Graph test_graph" in call_args[1]["content"]
+ assert result is not None
+ assert result["activity_status"] == "Agent completed execution."
+ assert result["correctness_score"] == 0.8
+ # The structured block should have been instantiated
+ assert mock_structured_block.called
class TestIntegration:
@@ -626,8 +709,8 @@ class TestIntegration:
) as mock_get_block, patch(
"backend.executor.activity_status_generator.Settings"
) as mock_settings, patch(
- "backend.executor.activity_status_generator.llm_call"
- ) as mock_llm_call, patch(
+ "backend.executor.activity_status_generator.AIStructuredResponseGeneratorBlock"
+ ) as mock_structured_block, patch(
"backend.executor.activity_status_generator.is_feature_enabled",
return_value=True,
):
@@ -635,15 +718,16 @@ class TestIntegration:
mock_get_block.side_effect = lambda block_id: mock_blocks.get(block_id)
mock_settings.return_value.secrets.openai_internal_api_key = "test_key"
- mock_response = LLMResponse(
- raw_response={},
- prompt=[],
- response=expected_activity,
- tool_calls=None,
- prompt_tokens=100,
- completion_tokens=30,
- )
- mock_llm_call.return_value = mock_response
+ # Mock the structured block to return our expected response
+ mock_instance = mock_structured_block.return_value
+
+ async def mock_run(*args, **kwargs):
+ yield "response", {
+ "activity_status": expected_activity,
+ "correctness_score": 0.3, # Low score since there was a failure
+ }
+
+ mock_instance.run = mock_run
result = await generate_activity_status_for_execution(
graph_exec_id="test_exec",
@@ -654,24 +738,14 @@ class TestIntegration:
user_id="test_user",
)
- assert result == expected_activity
-
- # Verify the correct data was passed to LLM
- llm_call_args = mock_llm_call.call_args
- prompt = llm_call_args[1]["prompt"]
-
- # Check system prompt
- assert prompt[0]["role"] == "system"
- assert "user's perspective" in prompt[0]["content"]
-
- # Check user prompt contains expected data
- user_content = prompt[1]["content"]
- assert "Test Integration Agent" in user_content
- assert "user-friendly terms" in user_content.lower()
+ assert result is not None
+ assert result["activity_status"] == expected_activity
+ assert result["correctness_score"] == 0.3
- # Verify that execution data is present in the prompt
- assert "{" in user_content # Should contain JSON data
- assert "overall_status" in user_content
+ # Verify the structured block was called
+ assert mock_structured_block.called
+ # The structured block should have been instantiated
+ mock_structured_block.assert_called_once()
@pytest.mark.asyncio
async def test_manager_integration_with_disabled_feature(
diff --git a/autogpt_platform/backend/backend/executor/manager.py b/autogpt_platform/backend/backend/executor/manager.py
index 5b433d4081..3863984039 100644
--- a/autogpt_platform/backend/backend/executor/manager.py
+++ b/autogpt_platform/backend/backend/executor/manager.py
@@ -696,7 +696,7 @@ class ExecutionProcessor:
exec_meta.status = status
# Activity status handling
- activity_status = asyncio.run_coroutine_threadsafe(
+ activity_response = asyncio.run_coroutine_threadsafe(
generate_activity_status_for_execution(
graph_exec_id=graph_exec.graph_exec_id,
graph_id=graph_exec.graph_id,
@@ -708,18 +708,21 @@ class ExecutionProcessor:
),
self.node_execution_loop,
).result(timeout=60.0)
- if activity_status is not None:
- exec_stats.activity_status = activity_status
- log_metadata.info(f"Generated activity status: {activity_status}")
+ if activity_response is not None:
+ exec_stats.activity_status = activity_response["activity_status"]
+ exec_stats.correctness_score = activity_response["correctness_score"]
+ log_metadata.info(
+ f"Generated activity status: {activity_response['activity_status']} "
+ f"(correctness: {activity_response['correctness_score']:.2f})"
+ )
else:
log_metadata.debug(
- "Activity status generation disabled, not setting field"
+ "Activity status generation disabled, not setting fields"
)
-
+ finally:
# Communication handling
self._handle_agent_run_notif(db_client, graph_exec, exec_stats)
- finally:
update_graph_execution_state(
db_client=db_client,
graph_exec_id=graph_exec.graph_exec_id,
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index 43ee6b4fc2..9ea38ba7bb 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -7821,6 +7821,11 @@
"anyOf": [{ "type": "string" }, { "type": "null" }],
"title": "Activity Status",
"description": "AI-generated summary of what the agent did"
+ },
+ "correctness_score": {
+ "anyOf": [{ "type": "number" }, { "type": "null" }],
+ "title": "Correctness Score",
+ "description": "AI-generated score (0.0-1.0) indicating how well the execution achieved its intended purpose"
}
},
"additionalProperties": true,