AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend): filter graph-only blocks from CoPilot's find_block results (#11892)
Nicholas Tindlee8fc8ee62347
diff --git a/autogpt_platform/backend/backend/api/features/chat/tools/find_block.py b/autogpt_platform/backend/backend/api/features/chat/tools/find_block.pyindex 7ca85961f9..f55cd567e8 100644--- a/autogpt_platform/backend/backend/api/features/chat/tools/find_block.py+++ b/autogpt_platform/backend/backend/api/features/chat/tools/find_block.py@@ -13,10 +13,32 @@ from backend.api.features.chat.tools.models import (NoResultsResponse,)from backend.api.features.store.hybrid_search import unified_hybrid_search-from backend.data.block import get_block+from backend.data.block import BlockType, get_blocklogger = logging.getLogger(__name__)+_TARGET_RESULTS = 10+# Over-fetch to compensate for post-hoc filtering of graph-only blocks.+# 40 is 2x current removed; speed of query 10 vs 40 is minimial+_OVERFETCH_PAGE_SIZE = 40++# Block types that only work within graphs and cannot run standalone in CoPilot.+COPILOT_EXCLUDED_BLOCK_TYPES = {+ BlockType.INPUT, # Graph interface definition - data enters via chat, not graph inputs+ BlockType.OUTPUT, # Graph interface definition - data exits via chat, not graph outputs+ BlockType.WEBHOOK, # Wait for external events - would hang forever in CoPilot+ BlockType.WEBHOOK_MANUAL, # Same as WEBHOOK+ BlockType.NOTE, # Visual annotation only - no runtime behavior+ BlockType.HUMAN_IN_THE_LOOP, # Pauses for human approval - CoPilot IS human-in-the-loop+ BlockType.AGENT, # AgentExecutorBlock requires execution_context - use run_agent tool+}++# Specific block IDs excluded from CoPilot (STANDARD type but still require graph context)+COPILOT_EXCLUDED_BLOCK_IDS = {+ # SmartDecisionMakerBlock - dynamically discovers downstream blocks via graph topology+ "3b191d9f-356f-482d-8238-ba04b6d18381",+}+class FindBlockTool(BaseTool):"""Tool for searching available blocks."""@@ -88,7 +110,7 @@ class FindBlockTool(BaseTool):query=query,content_types=[ContentType.BLOCK],page=1,- page_size=10,+ page_size=_OVERFETCH_PAGE_SIZE,)if not results:@@ -108,60 +130,90 @@ class FindBlockTool(BaseTool):block = get_block(block_id)# Skip disabled blocks- if block and not block.disabled:- # Get input/output schemas- input_schema = {}- output_schema = {}- try:- input_schema = block.input_schema.jsonschema()- except Exception:- pass- try:- output_schema = block.output_schema.jsonschema()- except Exception:- pass-- # Get categories from block instance- categories = []- if hasattr(block, "categories") and block.categories:- categories = [cat.value for cat in block.categories]-- # Extract required inputs for easier use- required_inputs: list[BlockInputFieldInfo] = []- if input_schema:- properties = input_schema.get("properties", {})- required_fields = set(input_schema.get("required", []))- # Get credential field names to exclude from required inputs- credentials_fields = set(- block.input_schema.get_credentials_fields().keys()- )+ if not block or block.disabled:+ continue++ # Skip blocks excluded from CoPilot (graph-only blocks)+ if (+ block.block_type in COPILOT_EXCLUDED_BLOCK_TYPES+ or block.id in COPILOT_EXCLUDED_BLOCK_IDS+ ):+ continue++ # Get input/output schemas+ input_schema = {}+ output_schema = {}+ try:+ input_schema = block.input_schema.jsonschema()+ except Exception as e:+ logger.debug(+ "Failed to generate input schema for block %s: %s",+ block_id,+ e,+ )+ try:+ output_schema = block.output_schema.jsonschema()+ except Exception as e:+ logger.debug(+ "Failed to generate output schema for block %s: %s",+ block_id,+ e,+ )- for field_name, field_schema in properties.items():- # Skip credential fields - they're handled separately- if field_name in credentials_fields:- continue-- required_inputs.append(- BlockInputFieldInfo(- name=field_name,- type=field_schema.get("type", "string"),- description=field_schema.get("description", ""),- required=field_name in required_fields,- default=field_schema.get("default"),- )- )+ # Get categories from block instance+ categories = []+ if hasattr(block, "categories") and block.categories:+ categories = [cat.value for cat in block.categories]++ # Extract required inputs for easier use+ required_inputs: list[BlockInputFieldInfo] = []+ if input_schema:+ properties = input_schema.get("properties", {})+ required_fields = set(input_schema.get("required", []))+ # Get credential field names to exclude from required inputs+ credentials_fields = set(+ block.input_schema.get_credentials_fields().keys()+ )- blocks.append(- BlockInfoSummary(- id=block_id,- name=block.name,- description=block.description or "",- categories=categories,- input_schema=input_schema,- output_schema=output_schema,- required_inputs=required_inputs,+ for field_name, field_schema in properties.items():+ # Skip credential fields - they're handled separately+ if field_name in credentials_fields:+ continue++ required_inputs.append(+ BlockInputFieldInfo(+ name=field_name,+ type=field_schema.get("type", "string"),+ description=field_schema.get("description", ""),+ required=field_name in required_fields,+ default=field_schema.get("default"),+ ))++ blocks.append(+ BlockInfoSummary(+ id=block_id,+ name=block.name,+ description=block.description or "",+ categories=categories,+ input_schema=input_schema,+ output_schema=output_schema,+ required_inputs=required_inputs,)+ )++ if len(blocks) >= _TARGET_RESULTS:+ break++ if blocks and len(blocks) < _TARGET_RESULTS:+ logger.debug(+ "find_block returned %d/%d results for query '%s' "+ "(filtered %d excluded/disabled blocks)",+ len(blocks),+ _TARGET_RESULTS,+ query,+ len(results) - len(blocks),+ )if not blocks:return NoResultsResponse(diff --git a/autogpt_platform/backend/backend/api/features/chat/tools/find_block_test.py b/autogpt_platform/backend/backend/api/features/chat/tools/find_block_test.pynew file mode 100644index 0000000000..0f3d4cbfa5--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/chat/tools/find_block_test.py@@ -0,0 +1,139 @@+"""Tests for block filtering in FindBlockTool."""++from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from backend.api.features.chat.tools.find_block import (+ COPILOT_EXCLUDED_BLOCK_IDS,+ COPILOT_EXCLUDED_BLOCK_TYPES,+ FindBlockTool,+)+from backend.api.features.chat.tools.models import BlockListResponse+from backend.data.block import BlockType++from ._test_data import make_session++_TEST_USER_ID = "test-user-find-block"+++def make_mock_block(+ block_id: str, name: str, block_type: BlockType, disabled: bool = False+):+ """Create a mock block for testing."""+ mock = MagicMock()+ mock.id = block_id+ mock.name = name+ mock.description = f"{name} description"+ mock.block_type = block_type+ mock.disabled = disabled+ mock.input_schema = MagicMock()+ mock.input_schema.jsonschema.return_value = {"properties": {}, "required": []}+ mock.input_schema.get_credentials_fields.return_value = {}+ mock.output_schema = MagicMock()+ mock.output_schema.jsonschema.return_value = {}+ mock.categories = []+ return mock+++class TestFindBlockFiltering:+ """Tests for block filtering in FindBlockTool."""++ def test_excluded_block_types_contains_expected_types(self):+ """Verify COPILOT_EXCLUDED_BLOCK_TYPES contains all graph-only types."""+ assert BlockType.INPUT in COPILOT_EXCLUDED_BLOCK_TYPES+ assert BlockType.OUTPUT in COPILOT_EXCLUDED_BLOCK_TYPES+ assert BlockType.WEBHOOK in COPILOT_EXCLUDED_BLOCK_TYPES+ assert BlockType.WEBHOOK_MANUAL in COPILOT_EXCLUDED_BLOCK_TYPES+ assert BlockType.NOTE in COPILOT_EXCLUDED_BLOCK_TYPES+ assert BlockType.HUMAN_IN_THE_LOOP in COPILOT_EXCLUDED_BLOCK_TYPES+ assert BlockType.AGENT in COPILOT_EXCLUDED_BLOCK_TYPES++ def test_excluded_block_ids_contains_smart_decision_maker(self):+ """Verify SmartDecisionMakerBlock is in COPILOT_EXCLUDED_BLOCK_IDS."""+ assert "3b191d9f-356f-482d-8238-ba04b6d18381" in COPILOT_EXCLUDED_BLOCK_IDS++ @pytest.mark.asyncio(loop_scope="session")+ async def test_excluded_block_type_filtered_from_results(self):+ """Verify blocks with excluded BlockTypes are filtered from search results."""+ session = make_session(user_id=_TEST_USER_ID)++ # Mock search returns an INPUT block (excluded) and a STANDARD block (included)+ search_results = [+ {"content_id": "input-block-id", "score": 0.9},+ {"content_id": "standard-block-id", "score": 0.8},+ ]++ input_block = make_mock_block("input-block-id", "Input Block", BlockType.INPUT)+ standard_block = make_mock_block(+ "standard-block-id", "HTTP Request", BlockType.STANDARD+ )++ def mock_get_block(block_id):+ return {+ "input-block-id": input_block,+ "standard-block-id": standard_block,+ }.get(block_id)++ with patch(+ "backend.api.features.chat.tools.find_block.unified_hybrid_search",+ new_callable=AsyncMock,+ return_value=(search_results, 2),+ ):+ with patch(+ "backend.api.features.chat.tools.find_block.get_block",+ side_effect=mock_get_block,+ ):+ tool = FindBlockTool()+ response = await tool._execute(+ user_id=_TEST_USER_ID, session=session, query="test"+ )++ # Should only return the standard block, not the INPUT block+ assert isinstance(response, BlockListResponse)+ assert len(response.blocks) == 1+ assert response.blocks[0].id == "standard-block-id"++ @pytest.mark.asyncio(loop_scope="session")+ async def test_excluded_block_id_filtered_from_results(self):+ """Verify SmartDecisionMakerBlock is filtered from search results."""+ session = make_session(user_id=_TEST_USER_ID)++ smart_decision_id = "3b191d9f-356f-482d-8238-ba04b6d18381"+ search_results = [+ {"content_id": smart_decision_id, "score": 0.9},+ {"content_id": "normal-block-id", "score": 0.8},+ ]++ # SmartDecisionMakerBlock has STANDARD type but is excluded by ID+ smart_block = make_mock_block(+ smart_decision_id, "Smart Decision Maker", BlockType.STANDARD+ )+ normal_block = make_mock_block(+ "normal-block-id", "Normal Block", BlockType.STANDARD+ )++ def mock_get_block(block_id):+ return {+ smart_decision_id: smart_block,+ "normal-block-id": normal_block,+ }.get(block_id)++ with patch(+ "backend.api.features.chat.tools.find_block.unified_hybrid_search",+ new_callable=AsyncMock,+ return_value=(search_results, 2),+ ):+ with patch(+ "backend.api.features.chat.tools.find_block.get_block",+ side_effect=mock_get_block,+ ):+ tool = FindBlockTool()+ response = await tool._execute(+ user_id=_TEST_USER_ID, session=session, query="decision"+ )++ # Should only return normal block, not SmartDecisionMakerBlock+ assert isinstance(response, BlockListResponse)+ assert len(response.blocks) == 1+ assert response.blocks[0].id == "normal-block-id"diff --git a/autogpt_platform/backend/backend/api/features/chat/tools/run_block.py b/autogpt_platform/backend/backend/api/features/chat/tools/run_block.pyindex 51bb2c0575..590f81ff23 100644--- a/autogpt_platform/backend/backend/api/features/chat/tools/run_block.py+++ b/autogpt_platform/backend/backend/api/features/chat/tools/run_block.py@@ -8,6 +8,10 @@ from typing import Anyfrom pydantic_core import PydanticUndefinedfrom backend.api.features.chat.model import ChatSession+from backend.api.features.chat.tools.find_block import (+ COPILOT_EXCLUDED_BLOCK_IDS,+ COPILOT_EXCLUDED_BLOCK_TYPES,+)from backend.data.block import get_blockfrom backend.data.execution import ExecutionContextfrom backend.data.model import CredentialsMetaInput@@ -212,6 +216,19 @@ class RunBlockTool(BaseTool):session_id=session_id,)+ # Check if block is excluded from CoPilot (graph-only blocks)+ if (+ block.block_type in COPILOT_EXCLUDED_BLOCK_TYPES+ or block.id in COPILOT_EXCLUDED_BLOCK_IDS+ ):+ return ErrorResponse(+ message=(+ f"Block '{block.name}' cannot be run directly in CoPilot. "+ "This block is designed for use within graphs only."+ ),+ session_id=session_id,+ )+logger.info(f"Executing block {block.name} ({block_id}) for user {user_id}")creds_manager = IntegrationCredentialsManager()diff --git a/autogpt_platform/backend/backend/api/features/chat/tools/run_block_test.py b/autogpt_platform/backend/backend/api/features/chat/tools/run_block_test.pynew file mode 100644index 0000000000..2aae45e875--- /dev/null+++ b/autogpt_platform/backend/backend/api/features/chat/tools/run_block_test.py@@ -0,0 +1,106 @@+"""Tests for block execution guards in RunBlockTool."""++from unittest.mock import MagicMock, patch++import pytest++from backend.api.features.chat.tools.models import ErrorResponse+from backend.api.features.chat.tools.run_block import RunBlockTool+from backend.data.block import BlockType++from ._test_data import make_session++_TEST_USER_ID = "test-user-run-block"+++def make_mock_block(+ block_id: str, name: str, block_type: BlockType, disabled: bool = False+):+ """Create a mock block for testing."""+ mock = MagicMock()+ mock.id = block_id+ mock.name = name+ mock.block_type = block_type+ mock.disabled = disabled+ mock.input_schema = MagicMock()+ mock.input_schema.jsonschema.return_value = {"properties": {}, "required": []}+ mock.input_schema.get_credentials_fields_info.return_value = []+ return mock+++class TestRunBlockFiltering:+ """Tests for block execution guards in RunBlockTool."""++ @pytest.mark.asyncio(loop_scope="session")+ async def test_excluded_block_type_returns_error(self):+ """Attempting to execute a block with excluded BlockType returns error."""+ session = make_session(user_id=_TEST_USER_ID)++ input_block = make_mock_block("input-block-id", "Input Block", BlockType.INPUT)++ with patch(+ "backend.api.features.chat.tools.run_block.get_block",+ return_value=input_block,+ ):+ tool = RunBlockTool()+ response = await tool._execute(+ user_id=_TEST_USER_ID,+ session=session,+ block_id="input-block-id",+ input_data={},+ )++ assert isinstance(response, ErrorResponse)+ assert "cannot be run directly in CoPilot" in response.message+ assert "designed for use within graphs only" in response.message++ @pytest.mark.asyncio(loop_scope="session")+ async def test_excluded_block_id_returns_error(self):+ """Attempting to execute SmartDecisionMakerBlock returns error."""+ session = make_session(user_id=_TEST_USER_ID)++ smart_decision_id = "3b191d9f-356f-482d-8238-ba04b6d18381"+ smart_block = make_mock_block(+ smart_decision_id, "Smart Decision Maker", BlockType.STANDARD+ )++ with patch(+ "backend.api.features.chat.tools.run_block.get_block",+ return_value=smart_block,+ ):+ tool = RunBlockTool()+ response = await tool._execute(+ user_id=_TEST_USER_ID,+ session=session,+ block_id=smart_decision_id,+ input_data={},+ )++ assert isinstance(response, ErrorResponse)+ assert "cannot be run directly in CoPilot" in response.message++ @pytest.mark.asyncio(loop_scope="session")+ async def test_non_excluded_block_passes_guard(self):+ """Non-excluded blocks pass the filtering guard (may fail later for other reasons)."""+ session = make_session(user_id=_TEST_USER_ID)++ standard_block = make_mock_block(+ "standard-id", "HTTP Request", BlockType.STANDARD+ )++ with patch(+ "backend.api.features.chat.tools.run_block.get_block",+ return_value=standard_block,+ ):+ tool = RunBlockTool()+ response = await tool._execute(+ user_id=_TEST_USER_ID,+ session=session,+ block_id="standard-id",+ input_data={},+ )++ # Should NOT be an ErrorResponse about CoPilot exclusion+ # (may be other errors like missing credentials, but not the exclusion guard)+ if isinstance(response, ErrorResponse):+ assert "cannot be run directly in CoPilot" not in response.messagediff --git a/autogpt_platform/backend/backend/data/execution_queue_test.py b/autogpt_platform/backend/backend/data/execution_queue_test.pyindex ffe0fb265b..7a76adfe05 100644--- a/autogpt_platform/backend/backend/data/execution_queue_test.py+++ b/autogpt_platform/backend/backend/data/execution_queue_test.py@@ -3,8 +3,6 @@import queueimport threading-import pytest-from backend.data.execution import ExecutionQueue