Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.
Python agentagentic-aiagentic-frameworkagentic-workflowaiautomationclaudedeepseekgenaigptllmlow-codemcpnextjsno-codeopenaipythonskillsworkflow
test: construct real LLM generation dependencies (#43174)
Asuka Minato38f1ecdd3b9c
diff --git a/api/tests/unit_tests/core/llm_generator/test_llm_generator.py b/api/tests/unit_tests/core/llm_generator/test_llm_generator.pyindex 98b51f2527c..c744d433356 100644--- a/api/tests/unit_tests/core/llm_generator/test_llm_generator.py+++ b/api/tests/unit_tests/core/llm_generator/test_llm_generator.py@@ -17,8 +17,9 @@ from core.llm_generator import llm_generator as llm_generator_modulefrom core.llm_generator.entities import RuleCodeGeneratePayload, RuleGeneratePayload, RuleStructuredOutputPayloadfrom core.llm_generator.llm_generator import LLMGeneratorfrom core.model_context import get_credit_usage_metadata-from core.model_manager import ModelInstance, ModelManager+from core.model_manager import ModelManagerfrom core.plugin.impl.base import _get_plugin_daemon_request_timeout+from core.plugin.impl.model_runtime_factory import create_plugin_model_managerfrom graphon.enums import WorkflowNodeExecutionStatusfrom graphon.model_runtime.entities.llm_entities import LLMMode, LLMResult, LLMUsagefrom graphon.model_runtime.entities.message_entities import AssistantPromptMessage@@ -32,6 +33,7 @@ from models.workflow import (WorkflowNodeExecutionTriggeredFrom,)from services.workflow_service import WorkflowService+from tests.unit_tests.core.model_fixtures import make_model_config, make_model_instance@pytest.fixture@@ -47,17 +49,14 @@ def database(sqlite_session: Session, monkeypatch: pytest.MonkeyPatch) -> Iterat@pytest.fixture-def recording_model_instance(monkeypatch: pytest.MonkeyPatch) -> Mock:- model_instance = Mock(spec=ModelInstance)- model_instance.invoke_llm.return_value = _llm_result('{"modified": "workflow"}')- model_manager = Mock(spec=ModelManager)- model_manager.get_model_instance.return_value = model_instance- monkeypatch.setattr(- llm_generator_module.ModelManager,- "for_tenant",- Mock(return_value=model_manager),- )- return model_instance+def llm_invocation(monkeypatch: pytest.MonkeyPatch) -> Mock:+ model_instance = make_model_instance(provider="openai", model="gpt-4")+ invocation = Mock(return_value=_llm_result('{"modified": "workflow"}'))+ monkeypatch.setattr(model_instance, "invoke_llm", invocation)+ model_manager = create_plugin_model_manager(tenant_id="tenant")+ monkeypatch.setattr(model_manager, "get_model_instance", Mock(return_value=model_instance))+ monkeypatch.setattr(llm_generator_module.ModelManager, "for_tenant", Mock(return_value=model_manager))+ return invocationdef _llm_result(content: str) -> LLMResult:@@ -470,11 +469,15 @@ class TestLLMGenerator:def test_prepared_suggested_questions_defer_provider_calls_without_resolving_again(self, monkeypatch: pytest.MonkeyPatch, use_configured_model: bool) -> None:- model_instance = Mock(spec=ModelInstance)- model_instance.get_model_schema.return_value.parameter_rules = []- manager = Mock(spec=ModelManager)- manager.get_model_instance.return_value = model_instance- manager.get_default_model_instance.return_value = model_instance+ model_instance = make_model_instance(provider="openai", model="custom-model")+ schema = make_model_config(provider="openai", model="custom-model", mode="chat").model_schema+ get_schema = Mock(return_value=schema)+ invocation = Mock()+ monkeypatch.setattr(model_instance, "get_model_schema", get_schema)+ monkeypatch.setattr(model_instance, "invoke_llm", invocation)+ manager = create_plugin_model_manager(tenant_id="tenant_id")+ monkeypatch.setattr(manager, "get_model_instance", Mock(return_value=model_instance))+ monkeypatch.setattr(manager, "get_default_model_instance", Mock(return_value=model_instance))resolve_manager = Mock(return_value=manager)monkeypatch.setattr(ModelManager, "for_tenant", resolve_manager)original_metadata = get_credit_usage_metadata()@@ -489,7 +492,7 @@ class TestLLMGenerator:assert timeout.read == 30.0return _llm_result('["Next question?"]')- model_instance.invoke_llm.side_effect = invoke+ invocation.side_effect = invokemodel_config = ({"provider": "openai",@@ -502,8 +505,8 @@ class TestLLMGenerator:prepared_model = LLMGenerator.prepare_suggested_questions_model("tenant_id", model_config=model_config)assert prepared_model is not None- model_instance.get_model_schema.assert_not_called()- model_instance.invoke_llm.assert_not_called()+ get_schema.assert_not_called()+ invocation.assert_not_called()resolve_manager.side_effect = AssertionError("Model lookup must finish in the preparation phase")result = LLMGenerator.invoke_suggested_questions_after_answer(@@ -511,7 +514,7 @@ class TestLLMGenerator:)assert result == ["Next question?"]- parameters = model_instance.invoke_llm.call_args.kwargs+ parameters = invocation.call_args.kwargsassert parameters["model_parameters"] == ({"temperature": 0.2} if use_configured_model else {"max_tokens": 256, "temperature": 0.0})@@ -662,9 +665,7 @@ class TestLLMGenerator:assert "Random error" in result["error"]def test_generate_qa_document_success(self, mock_model_instance):- mock_response = MagicMock(spec=LLMResult)- mock_response.message = MagicMock()- mock_response.message.get_text_content.return_value = "QA Document Content"+ mock_response = _llm_result("QA Document Content")mock_model_instance.invoke_llm.return_value = mock_responseresult = LLMGenerator.generate_qa_document("tenant_id", "query", "English")@@ -723,11 +724,11 @@ class TestLLMGenerator:def test_instruction_modify_legacy_without_last_run_uses_real_empty_query(self,database: Session,- recording_model_instance: Mock,+ llm_invocation: Mock,model_config_entity: ModelConfig,):app = _persist_app(database)- recording_model_instance.invoke_llm.return_value = _llm_result('{"modified": "prompt"}')+ llm_invocation.return_value = _llm_result('{"modified": "prompt"}')result = LLMGenerator.instruction_modify_legacy(app.tenant_id,@@ -740,14 +741,14 @@ class TestLLMGenerator:)assert result == {"modified": "prompt"}- user_payload = json.loads(recording_model_instance.invoke_llm.call_args.kwargs["prompt_messages"][1].content)+ user_payload = json.loads(llm_invocation.call_args.kwargs["prompt_messages"][1].content)assert "null" in user_payload["instruction"]assert "current_val" in user_payload["instruction"]def test_instruction_modify_legacy_reads_latest_tenant_scoped_message(self,database: Session,- recording_model_instance: Mock,+ llm_invocation: Mock,model_config_entity: ModelConfig,):app = _persist_app(database)@@ -772,14 +773,14 @@ class TestLLMGenerator:query="other tenant question",created_at=datetime(2026, 1, 3),)- recording_model_instance.invoke_llm.return_value = _llm_result('{"modified": "prompt"}')+ llm_invocation.return_value = _llm_result('{"modified": "prompt"}')result = LLMGenerator.instruction_modify_legacy(app.tenant_id, app.id, "current", "instruction", model_config_entity, "ideal", session=database)assert result == {"modified": "prompt"}- user_payload = json.loads(recording_model_instance.invoke_llm.call_args.kwargs["prompt_messages"][1].content)+ user_payload = json.loads(llm_invocation.call_args.kwargs["prompt_messages"][1].content)assert user_payload["last_run"]["query"] == "latest question"assert user_payload["last_run"]["answer"] == "latest answer"assert "older question" not in json.dumps(user_payload)@@ -854,7 +855,7 @@ class TestLLMGenerator:self,database: Session,sqlite_session_factory: sessionmaker[Session],- recording_model_instance: Mock,+ llm_invocation: Mock,model_config_entity: ModelConfig,):app = _persist_app(database)@@ -881,7 +882,7 @@ class TestLLMGenerator:)assert result == {"modified": "workflow"}- user_payload = json.loads(recording_model_instance.invoke_llm.call_args.kwargs["prompt_messages"][1].content)+ user_payload = json.loads(llm_invocation.call_args.kwargs["prompt_messages"][1].content)assert user_payload["last_run"]["inputs"] == {"input": "value"}assert user_payload["last_run"]["agent_log"][0]["data"] == {"step": 1}@@ -889,7 +890,7 @@ class TestLLMGenerator:self,database: Session,sqlite_session_factory: sessionmaker[Session],- recording_model_instance: Mock,+ llm_invocation: Mock,model_config_entity: ModelConfig,):app = _persist_app(database)@@ -910,7 +911,7 @@ class TestLLMGenerator:)assert result == {"modified": "workflow"}- user_payload = json.loads(recording_model_instance.invoke_llm.call_args.kwargs["prompt_messages"][1].content)+ user_payload = json.loads(llm_invocation.call_args.kwargs["prompt_messages"][1].content)assert user_payload["last_run"]["agent_log"] == []@pytest.mark.parametrize(@@ -924,14 +925,14 @@ class TestLLMGenerator:self,database: Session,sqlite_session_factory: sessionmaker[Session],- recording_model_instance: Mock,+ llm_invocation: Mock,model_config_entity: ModelConfig,node_type: str | None,):app = _persist_app(database)_persist_workflow(database, app, node_type=node_type)workflow_service = WorkflowService(sqlite_session_factory)- recording_model_instance.invoke_llm.return_value = _llm_result('{"modified": "fallback"}')+ llm_invocation.return_value = _llm_result('{"modified": "fallback"}')result = LLMGenerator.instruction_modify_workflow(app.tenant_id,@@ -951,7 +952,7 @@ class TestLLMGenerator:self,database: Session,sqlite_session_factory: sessionmaker[Session],- recording_model_instance: Mock,+ llm_invocation: Mock,model_config_entity: ModelConfig,):app = _persist_app(database)@@ -971,7 +972,7 @@ class TestLLMGenerator:)assert result == {"modified": "workflow"}- system_prompt = recording_model_instance.invoke_llm.call_args.kwargs["prompt_messages"][0].content+ system_prompt = llm_invocation.call_args.kwargs["prompt_messages"][0].contentassert system_prompt == llm_generator_module.LLM_MODIFY_PROMPT_SYSTEM@pytest.mark.parametrize(@@ -1032,12 +1033,12 @@ class TestPromptGenerationTelemetryEmit:PromptGenerationEvent so the metric pipeline is not accidentally broken."""@pytest.fixture(autouse=True)- def _setup(self, recording_model_instance: Mock) -> None:- self.model_instance = recording_model_instance+ def _setup(self, llm_invocation: Mock) -> None:+ self.llm_invocation = llm_invocation@patch("core.llm_generator.llm_generator.telemetry_emit")def test_generate_code_emits_on_success(self, mock_emit: MagicMock) -> None:- self.model_instance.invoke_llm.return_value = _llm_result("print('hello')")+ self.llm_invocation.return_value = _llm_result("print('hello')")model_cfg = ModelConfig(provider="openai", name="gpt-4", mode="chat", completion_params={})args = RuleCodeGeneratePayload(instruction="write hello world", model_config=model_cfg, code_language="python")@@ -1055,7 +1056,7 @@ class TestPromptGenerationTelemetryEmit:@patch("core.llm_generator.llm_generator.telemetry_emit")def test_generate_code_emits_on_failure(self, mock_emit: MagicMock) -> None:- self.model_instance.invoke_llm.side_effect = InvokeError("model down")+ self.llm_invocation.side_effect = InvokeError("model down")model_cfg = ModelConfig(provider="openai", name="gpt-4", mode="chat", completion_params={})args = RuleCodeGeneratePayload(instruction="write hello world", model_config=model_cfg, code_language="python")@@ -1068,7 +1069,7 @@ class TestPromptGenerationTelemetryEmit:@patch("core.llm_generator.llm_generator.telemetry_emit")def test_generate_rule_config_no_variable_emits(self, mock_emit: MagicMock) -> None:- self.model_instance.invoke_llm.return_value = _llm_result("generated prompt")+ self.llm_invocation.return_value = _llm_result("generated prompt")model_cfg = ModelConfig(provider="anthropic", name="claude-3", mode="chat", completion_params={})args = RuleGeneratePayload(instruction="be helpful", model_config=model_cfg, no_variable=True)@@ -1082,7 +1083,7 @@ class TestPromptGenerationTelemetryEmit:@patch("core.llm_generator.llm_generator.telemetry_emit")def test_generate_rule_config_no_variable_emits_on_failure(self, mock_emit: MagicMock) -> None:- self.model_instance.invoke_llm.side_effect = InvokeError("auth fail")+ self.llm_invocation.side_effect = InvokeError("auth fail")model_cfg = ModelConfig(provider="anthropic", name="claude-3", mode="chat", completion_params={})args = RuleGeneratePayload(instruction="be helpful", model_config=model_cfg, no_variable=True)diff --git a/api/tests/unit_tests/core/llm_generator/test_llm_generator_missing.py b/api/tests/unit_tests/core/llm_generator/test_llm_generator_missing.pyindex 6fbf18afbbb..8631c7fb752 100644--- a/api/tests/unit_tests/core/llm_generator/test_llm_generator_missing.py+++ b/api/tests/unit_tests/core/llm_generator/test_llm_generator_missing.py@@ -8,13 +8,15 @@ from sqlalchemy.orm import Session, sessionmakerimport core.llm_generator.llm_generator as generator_modulefrom core.llm_generator.llm_generator import LLMGenerator, _parse_string_list-from core.model_manager import ModelInstance, ModelManager+from core.model_manager import ModelManager+from core.plugin.impl.model_runtime_factory import create_plugin_model_managerfrom core.workflow.generator import tool_catalogue as tool_catalogue_modulefrom core.workflow.generator.tool_catalogue import ToolCatalogueEntryfrom graphon.model_runtime.entities.llm_entities import LLMResult, LLMUsagefrom graphon.model_runtime.entities.message_entities import AssistantPromptMessagefrom models.dataset import Datasetfrom services.workflow_service import WorkflowService+from tests.unit_tests.core.model_fixtures import make_model_instancefrom tests.unit_tests.model_factories import make_dataset@@ -36,13 +38,15 @@ def _llm_result(content: str) -> LLMResult:)-def _model_manager() -> tuple[MagicMock, MagicMock]:- """Build spec-constrained mocks for the model-manager boundary and its default model."""-- model_manager = MagicMock(spec=ModelManager)- model_instance = MagicMock(spec=ModelInstance)- model_manager.get_default_model_instance.return_value = model_instance- return model_manager, model_instance+def _model_manager(monkeypatch: pytest.MonkeyPatch) -> tuple[ModelManager, MagicMock, MagicMock]:+ """Compose real model objects with controllable resolution and invocation boundaries."""+ model_manager = create_plugin_model_manager(tenant_id="tenant")+ model_instance = make_model_instance(provider="openai", model="gpt-4")+ invocation = MagicMock()+ resolve_default = MagicMock(return_value=model_instance)+ monkeypatch.setattr(model_instance, "invoke_llm", invocation)+ monkeypatch.setattr(model_manager, "get_default_model_instance", resolve_default)+ return model_manager, invocation, resolve_defaultdef _dataset(*, dataset_id: str, tenant_id: str, name: str, created_at: datetime) -> Dataset:@@ -84,48 +88,48 @@ class TestParseStringList:class TestGenerateWorkflowInstructionSuggestions:@patch("core.llm_generator.llm_generator.ModelManager.for_tenant")- def test_no_default_model(self, mock_for_tenant):- model_manager, _ = _model_manager()- model_manager.get_default_model_instance.side_effect = RuntimeError("no default model")+ def test_no_default_model(self, mock_for_tenant, monkeypatch: pytest.MonkeyPatch):+ model_manager, _, resolve_default = _model_manager(monkeypatch)+ resolve_default.side_effect = RuntimeError("no default model")mock_for_tenant.return_value = model_managerassert LLMGenerator.generate_workflow_instruction_suggestions("tenant", mode="workflow") == []@patch("core.llm_generator.llm_generator.ModelManager.for_tenant")@patch("core.llm_generator.llm_generator.LLMGenerator._build_suggestion_context")- def test_llm_success(self, mock_build_context, mock_for_tenant):+ def test_llm_success(self, mock_build_context, mock_for_tenant, monkeypatch: pytest.MonkeyPatch):mock_build_context.return_value = "context"- model_manager, model_instance = _model_manager()- model_instance.invoke_llm.return_value = _llm_result('["idea 1", "idea 2"]')+ model_manager, invocation, _ = _model_manager(monkeypatch)+ invocation.return_value = _llm_result('["idea 1", "idea 2"]')mock_for_tenant.return_value = model_managerresult = LLMGenerator.generate_workflow_instruction_suggestions("tenant", mode="workflow")assert result == ["idea 1", "idea 2"]- model_instance.invoke_llm.assert_called_once()+ invocation.assert_called_once()@patch("core.llm_generator.llm_generator.ModelManager.for_tenant")@patch("core.llm_generator.llm_generator.LLMGenerator._build_suggestion_context")- def test_llm_error(self, mock_build_context, mock_for_tenant):+ def test_llm_error(self, mock_build_context, mock_for_tenant, monkeypatch: pytest.MonkeyPatch):mock_build_context.return_value = "context"- model_manager, model_instance = _model_manager()- model_instance.invoke_llm.side_effect = RuntimeError("API error")+ model_manager, invocation, _ = _model_manager(monkeypatch)+ invocation.side_effect = RuntimeError("API error")mock_for_tenant.return_value = model_managerresult = LLMGenerator.generate_workflow_instruction_suggestions("tenant", mode="workflow")assert result == []- model_instance.invoke_llm.assert_called_once()+ invocation.assert_called_once()@patch("core.llm_generator.llm_generator.ModelManager.for_tenant")@patch("core.llm_generator.llm_generator.LLMGenerator._build_suggestion_context")- def test_llm_bad_output(self, mock_build_context, mock_for_tenant):+ def test_llm_bad_output(self, mock_build_context, mock_for_tenant, monkeypatch: pytest.MonkeyPatch):mock_build_context.return_value = "context"- model_manager, model_instance = _model_manager()- model_instance.invoke_llm.return_value = _llm_result("Not a list")+ model_manager, invocation, _ = _model_manager(monkeypatch)+ invocation.return_value = _llm_result("Not a list")mock_for_tenant.return_value = model_managerresult = LLMGenerator.generate_workflow_instruction_suggestions("tenant", mode="workflow")assert result == []- model_instance.invoke_llm.assert_called_once()+ invocation.assert_called_once()@pytest.mark.parametrize("sqlite_session", [(Dataset,)], indirect=True)diff --git a/api/tests/unit_tests/core/model_fixtures.py b/api/tests/unit_tests/core/model_fixtures.pyindex ab4ddf8ee40..93f6c9c4af3 100644--- a/api/tests/unit_tests/core/model_fixtures.py+++ b/api/tests/unit_tests/core/model_fixtures.py@@ -59,3 +59,11 @@ def make_token_buffer_memory(config: ModelConfigWithCredentialsEntity) -> TokenBcredentials=config.credentials,),)+++def make_model_instance(*, provider: str, model: str) -> ModelInstance:+ """Build an LLM instance with explicit credentials and no provider lookup."""+ config = make_model_config(provider=provider, model=model, mode="chat")+ return ModelInstance(+ provider_model_bundle=config.provider_model_bundle, model=config.model, credentials=config.credentials+ )diff --git a/api/tests/unit_tests/core/prompt/test_agent_history_prompt_transform.py b/api/tests/unit_tests/core/prompt/test_agent_history_prompt_transform.pyindex 28966242d81..4e4853fe122 100644--- a/api/tests/unit_tests/core/prompt/test_agent_history_prompt_transform.py+++ b/api/tests/unit_tests/core/prompt/test_agent_history_prompt_transform.py@@ -1,10 +1,7 @@from unittest.mock import MagicMock-from core.app.entities.app_invoke_entities import (- ModelConfigWithCredentialsEntity,-)-from core.entities.provider_configuration import ProviderModelBundle-from core.memory.token_buffer_memory import TokenBufferMemory+from pytest_mock import MockerFixture+from core.prompt.agent_history_prompt_transform import AgentHistoryPromptTransformfrom graphon.model_runtime.entities.message_entities import (AssistantPromptMessage,@@ -12,11 +9,10 @@ from graphon.model_runtime.entities.message_entities import (ToolPromptMessage,UserPromptMessage,)-from graphon.model_runtime.model_providers.base.large_language_model import LargeLanguageModel-from models.model import Conversation+from tests.unit_tests.core.model_fixtures import make_model_config, make_token_buffer_memory-def test_get_prompt():+def test_get_prompt(mocker: MockerFixture):prompt_messages = [SystemPromptMessage(content="System Template"),UserPromptMessage(content="User Query"),@@ -40,21 +36,14 @@ def test_get_prompt():def side_effect_get_num_tokens(*args):return len(args[2])- large_language_model_mock = MagicMock(spec=LargeLanguageModel)- large_language_model_mock.get_num_tokens = MagicMock(side_effect=side_effect_get_num_tokens)-- provider_model_bundle_mock = MagicMock(spec=ProviderModelBundle)- provider_model_bundle_mock.model_type_instance = large_language_model_mock-- model_config_mock = MagicMock(spec=ModelConfigWithCredentialsEntity)- model_config_mock.model = "openai"- model_config_mock.credentials = {}- model_config_mock.provider_model_bundle = provider_model_bundle_mock-- memory = TokenBufferMemory(conversation=Conversation(), model_instance=model_config_mock)+ model_config = make_model_config(provider="openai", model="gpt-4", mode="chat")+ mocker.patch.object(+ model_config.provider_model_bundle.model_type_instance, "get_num_tokens", side_effect=side_effect_get_num_tokens+ )+ memory = make_token_buffer_memory(model_config)transform = AgentHistoryPromptTransform(- model_config=model_config_mock,+ model_config=model_config,prompt_messages=prompt_messages,history_messages=history_messages,memory=memory,