Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: use SQLite sessions in rag pipeline dsl (#39053)

Asuka Minato6e9a3f8654e9
diff --git a/api/tests/unit_tests/services/rag_pipeline/pipeline_template/conftest.py b/api/tests/unit_tests/services/rag_pipeline/pipeline_template/conftest.py
deleted file mode 100644
index 2e65be48086..00000000000
--- a/api/tests/unit_tests/services/rag_pipeline/pipeline_template/conftest.py
+++ /dev/null
@@ -1,18 +0,0 @@
-from collections.abc import Callable
-from unittest.mock import Mock
-
-import pytest
-from pytest_mock import MockerFixture
-
-
-def patch_session_factory(mocker: MockerFixture) -> Callable[[str, Mock], Mock]:
- def _patch(module_path: str, session_mock: Mock) -> Mock:
- session_context = mocker.MagicMock()
- session_context.__enter__.return_value = session_mock
- session_context.__exit__.return_value = None
- session_maker = mocker.Mock(return_value=session_context)
- mocker.patch(f"{module_path}.session_factory.get_session_maker", return_value=session_maker)
- return session_maker
-
- return _patch
diff --git a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_dsl_service.py b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_dsl_service.py
index 5cdb2afd093..0b311b28c28 100644
--- a/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_dsl_service.py
+++ b/api/tests/unit_tests/services/rag_pipeline/test_rag_pipeline_dsl_service.py
@@ -1,25 +1,175 @@
+"""SQLite-backed tests for the RAG pipeline DSL service.
+
+Database behavior and transaction ownership use the shared SQLite fixtures. Plugin,
+Redis, validation, and HTTP boundaries remain isolated so each test can exercise the
+service branch that owns the behavior under test.
+"""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Generator
+from contextlib import contextmanager
from types import SimpleNamespace
-from typing import Any, cast
-from unittest.mock import MagicMock, Mock
+from typing import Any
+from unittest.mock import MagicMock, Mock, call
import pytest
import yaml
-from pytest_mock import MockerFixture
-from sqlalchemy.orm import Session
+from sqlalchemy import Engine, event, select
+from sqlalchemy.orm import Session, sessionmaker
from core.workflow.nodes.knowledge_index import KNOWLEDGE_INDEX_NODE_TYPE
from graphon.enums import BuiltinNodeTypes
+from models import Account, Tenant
+from models.dataset import (
+ Dataset,
+ Pipeline,
+ PipelineCustomizedTemplate,
+)
+from models.enums import DataSourceType
+from models.workflow import Workflow, WorkflowKind, WorkflowType
from services.dsl_version import check_version_compatibility
from services.entities.knowledge_entities.rag_pipeline_entities import IconInfo, RagPipelineDatasetCreateEntity
-from services.rag_pipeline import rag_pipeline_dsl_service
+from services.rag_pipeline import rag_pipeline_dsl_service as module
from services.rag_pipeline.rag_pipeline_dsl_service import (
+ ImportMode,
ImportStatus,
RagPipelineDslService,
+ RagPipelinePendingData,
)
+def service(sqlite_session: Session) -> RagPipelineDslService:
+ return RagPipelineDslService(session=sqlite_session)
+
+
+def _account(*, tenant_id: str = "tenant-1") -> Account:
+ tenant = Tenant(name="Tenant")
+ tenant.id = tenant_id
+ account = Account(name="Account", email="[email protected]")
+ account.id = "account-1"
+ account._current_tenant = tenant
+ return account
+
+
+def _pipeline(
+ session: Session,
+ *,
+ tenant_id: str = "tenant-1",
+ name: str = "Pipeline",
+ published: bool = False,
+) -> Pipeline:
+ pipeline = Pipeline(
+ tenant_id=tenant_id,
+ name=name,
+ description="description",
+ is_published=published,
+ created_by="account-1",
+ updated_by="account-1",
+ )
+ session.add(pipeline)
+ session.commit()
+ return pipeline
+
+
+def _workflow(session: Session, pipeline: Pipeline, *, graph: dict[str, Any] | None = None) -> Workflow:
+ workflow = Workflow(
+ id=f"workflow-{pipeline.id}",
+ tenant_id=pipeline.tenant_id,
+ app_id=pipeline.id,
+ type=WorkflowType.RAG_PIPELINE,
+ kind=WorkflowKind.STANDARD,
+ version=Workflow.VERSION_DRAFT,
+ graph=json.dumps(graph or {"nodes": [], "edges": []}),
+ features="{}",
+ created_by="account-1",
+ environment_variables=[],
+ conversation_variables=[],
+ rag_pipeline_variables=[],
+ )
+ pipeline.workflow_id = workflow.id
+ session.add(workflow)
+ session.commit()
+ return workflow
+
+
+def _dataset(
+ session: Session,
+ pipeline: Pipeline,
+ *,
+ name: str = "Dataset",
+ chunk_structure: str = "text_model",
+) -> Dataset:
+ dataset = Dataset(
+ tenant_id=pipeline.tenant_id,
+ name=name,
+ description="description",
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ indexing_technique="high_quality",
+ created_by="account-1",
+ maintainer="account-1",
+ chunk_structure=chunk_structure,
+ pipeline_id=pipeline.id,
+ icon_info={"icon": "📙", "icon_type": "emoji"},
+ )
+ session.add(dataset)
+ session.commit()
+ return dataset
+
+
+def _knowledge_configuration() -> SimpleNamespace:
+ return SimpleNamespace(
+ indexing_technique="high_quality",
+ embedding_model="text-embedding",
+ embedding_model_provider="openai",
+ chunk_structure="text_model",
+ retrieval_model=SimpleNamespace(model_dump=lambda: {}),
+ summary_index_setting=None,
+ keyword_number=10,
+ )
+
+
+def _valid_dsl(*, version: str = "0.1.0", name: str = "Imported") -> str:
+ return f"""
+version: {version}
+kind: rag_pipeline
+rag_pipeline:
+ name: {name}
+ description: description
+workflow:
+ graph:
+ nodes:
+ - id: knowledge-index
+ data:
+ type: {KNOWLEDGE_INDEX_NODE_TYPE}
+ edges: []
+"""
+
+
+@contextmanager
+def _raise_on_workflow_insert(engine: Engine) -> Generator[None]:
+ def raise_error(
+ _conn: object,
+ _cursor: object,
+ statement: str,
+ _parameters: object,
+ _context: object,
+ _executemany: object,
+ ) -> None:
+ if statement.lstrip().upper().startswith("INSERT") and "workflows" in statement:
+ raise RuntimeError("forced workflow INSERT")
+
+ event.listen(engine, "before_cursor_execute", raise_error)
+ try:
+ yield
+ finally:
+ event.remove(engine, "before_cursor_execute", raise_error)
+
+
@pytest.mark.parametrize(
- ("imported_version", "expected_status"),
+ ("version", "expected"),
[
("invalid", ImportStatus.FAILED),
("1.0.0", ImportStatus.PENDING),
@@ -27,1500 +177,700 @@ from services.rag_pipeline.rag_pipeline_dsl_service import (
("0.1.0", ImportStatus.COMPLETED),
],
)
-def test_check_version_compatibility(imported_version: str, expected_status: ImportStatus) -> None:
- assert (
- check_version_compatibility(imported_version, rag_pipeline_dsl_service.CURRENT_DSL_VERSION) == expected_status
- )
-
+def test_version_compatibility(version: str, expected: ImportStatus) -> None:
+ assert check_version_compatibility(version, module.CURRENT_DSL_VERSION) == expected
-def test_encrypt_decrypt_dataset_id_roundtrip() -> None:
- service = RagPipelineDslService(session=Mock())
+def test_dataset_id_encryption_roundtrip_and_invalid(service: RagPipelineDslService) -> None:
encrypted = service.encrypt_dataset_id("dataset-1", "tenant-1")
- decrypted = service.decrypt_dataset_id(encrypted, "tenant-1")
+ assert service.decrypt_dataset_id(encrypted, "tenant-1") == "dataset-1"
+ assert service.decrypt_dataset_id("not-base64", "tenant-1") is None
- assert decrypted == "dataset-1"
+def test_dependency_helpers_keep_plugin_analysis_external(
+ monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService
+) -> None:
+ assert service.get_leaked_dependencies("tenant-1", []) == []
+ dependency = MagicMock()
+ leaked = [MagicMock()]
+ analyze = Mock(return_value=leaked)
+ monkeypatch.setattr(module.DependenciesAnalysisService, "get_leaked_dependencies", analyze)
+ assert service.get_leaked_dependencies("tenant-1", [dependency]) == leaked
+ assert service._extract_dependencies_from_model_config({}) == []
+ assert service._extract_dependencies_from_workflow_graph({}) == []
-def test_decrypt_dataset_id_returns_none_for_invalid_payload() -> None:
- service = RagPipelineDslService(session=Mock())
- result = service.decrypt_dataset_id("not-base64", "tenant-1")
+def test_extract_dependencies_from_model_config_covers_models_rerankers_and_tools(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ def model_dependency(provider: str) -> str:
+ return f"model:{provider}"
- assert result is None
+ def tool_dependency(provider: str) -> str:
+ return f"tool:{provider}"
+ analyze_model = Mock(side_effect=model_dependency)
+ analyze_tool = Mock(side_effect=tool_dependency)
+ monkeypatch.setattr(module.DependenciesAnalysisService, "analyze_model_provider_dependency", analyze_model)
+ monkeypatch.setattr(module.DependenciesAnalysisService, "analyze_tool_dependency", analyze_tool)
+ model_config: dict[str, Any] = {
+ "model": {"provider": "openai"},
+ "dataset_configs": {
+ "datasets": {
+ "datasets": [
+ {
+ "reranking_model": {
+ "reranking_provider_name": {"provider": "cohere"},
+ }
+ }
+ ]
+ }
+ },
+ "agent_mode": {"tools": [{"provider_id": "google"}]},
+ }
-def test_get_leaked_dependencies_returns_empty_list_for_empty_input() -> None:
- result = RagPipelineDslService.get_leaked_dependencies("tenant-1", [])
+ dependencies = RagPipelineDslService._extract_dependencies_from_model_config(model_config)
- assert result == []
+ assert dependencies == ["model:openai", "model:cohere", "tool:google"]
+ assert analyze_model.call_args_list == [call("openai"), call("cohere")]
+ analyze_tool.assert_called_once_with("google")
-def test_get_leaked_dependencies_delegates_to_analysis_service(mocker: MockerFixture) -> None:
- expected = [Mock()]
- get_leaked_mock = mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.get_leaked_dependencies",
- return_value=expected,
+def test_extract_dependencies_from_workflow_graph_covers_plugin_and_model_nodes(
+ monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService
+) -> None:
+ def model_dependency(provider: str) -> str:
+ return f"model:{provider}"
+
+ def tool_dependency(provider: str) -> str:
+ return f"tool:{provider}"
+
+ monkeypatch.setattr(
+ module.DependenciesAnalysisService,
+ "analyze_model_provider_dependency",
+ Mock(side_effect=model_dependency),
+ )
+ monkeypatch.setattr(
+ module.DependenciesAnalysisService,
+ "analyze_tool_dependency",
+ Mock(side_effect=tool_dependency),
+ )
+ monkeypatch.setattr(
+ module.ToolNodeData,
+ "model_validate",
+ Mock(return_value=SimpleNamespace(provider_id="tool-provider")),
+ )
+ monkeypatch.setattr(
+ module.DatasourceNodeData,
+ "model_validate",
+ Mock(
+ side_effect=[
+ SimpleNamespace(provider_type="online_document", plugin_id="datasource-plugin"),
+ SimpleNamespace(provider_type="local_file", plugin_id="ignored-local-file"),
+ ]
+ ),
)
+ monkeypatch.setattr(
+ module.LLMNodeData,
+ "model_validate",
+ Mock(return_value=SimpleNamespace(model=SimpleNamespace(provider="llm-provider"))),
+ )
+ monkeypatch.setattr(
+ module.QuestionClassifierNodeData,
+ "model_validate",
+ Mock(return_value=SimpleNamespace(model=SimpleNamespace(provider="classifier-provider"))),
+ )
+ monkeypatch.setattr(
+ module.ParameterExtractorNodeData,
+ "model_validate",
+ Mock(return_value=SimpleNamespace(model=SimpleNamespace(provider="extractor-provider"))),
+ )
+ graph: dict[str, Any] = {
+ "nodes": [
+ {"data": {"type": BuiltinNodeTypes.TOOL}},
+ {"data": {"type": BuiltinNodeTypes.DATASOURCE}},
+ {"data": {"type": BuiltinNodeTypes.DATASOURCE}},
+ {"data": {"type": BuiltinNodeTypes.LLM}},
+ {"data": {"type": BuiltinNodeTypes.QUESTION_CLASSIFIER}},
+ {"data": {"type": BuiltinNodeTypes.PARAMETER_EXTRACTOR}},
+ {"data": {"type": "unknown"}},
+ ]
+ }
+
+ dependencies = service._extract_dependencies_from_workflow_graph(graph)
- dependency = Mock()
- result = RagPipelineDslService.get_leaked_dependencies("tenant-1", [dependency])
+ assert dependencies == [
+ "tool:tool-provider",
+ "datasource-plugin",
+ "model:llm-provider",
+ "model:classifier-provider",
+ "model:extractor-provider",
+ ]
- assert result == expected
- get_leaked_mock.assert_called_once_with(tenant_id="tenant-1", dependencies=[dependency])
+def test_extract_dependencies_from_workflow_graph_covers_knowledge_variants(
+ monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService
+) -> None:
+ def model_dependency(provider: str) -> str:
+ return f"model:{provider}"
+
+ monkeypatch.setattr(
+ module.DependenciesAnalysisService,
+ "analyze_model_provider_dependency",
+ Mock(side_effect=model_dependency),
+ )
+ knowledge_config = SimpleNamespace(
+ indexing_technique="high_quality",
+ embedding_model_provider="embedding-provider",
+ retrieval_model=SimpleNamespace(
+ reranking_mode="reranking_model",
+ reranking_enable=True,
+ reranking_model=SimpleNamespace(reranking_provider_name="knowledge-reranker"),
+ ),
+ )
+ monkeypatch.setattr(module.KnowledgeConfiguration, "model_validate", Mock(return_value=knowledge_config))
+ retrieval_configs = [
+ SimpleNamespace(
+ retrieval_mode="multiple",
+ multiple_retrieval_config=SimpleNamespace(
+ reranking_mode="weighted_score",
+ weights=SimpleNamespace(vector_setting=SimpleNamespace(embedding_provider_name="weighted-embedding")),
+ ),
+ ),
+ SimpleNamespace(
+ retrieval_mode="multiple",
+ multiple_retrieval_config=SimpleNamespace(
+ reranking_mode="reranking_model",
+ reranking_model=SimpleNamespace(provider="retrieval-reranker"),
+ ),
+ ),
+ SimpleNamespace(
+ retrieval_mode="single",
+ single_retrieval_config=SimpleNamespace(model=SimpleNamespace(provider="single-provider")),
+ ),
+ SimpleNamespace(retrieval_mode="multiple", multiple_retrieval_config=None),
+ SimpleNamespace(retrieval_mode="single", single_retrieval_config=None),
+ ]
+ monkeypatch.setattr(
+ module.KnowledgeRetrievalNodeData,
+ "model_validate",
+ Mock(side_effect=retrieval_configs),
+ )
+ graph: dict[str, Any] = {
+ "nodes": [
+ {"data": {"type": KNOWLEDGE_INDEX_NODE_TYPE}},
+ *[{"data": {"type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL}} for _ in retrieval_configs],
+ ]
+ }
+
+ dependencies = service._extract_dependencies_from_workflow_graph(graph)
-# --- check_dependencies ---
+ assert dependencies == [
+ "model:embedding-provider",
+ "model:knowledge-reranker",
+ "model:weighted-embedding",
+ "model:retrieval-reranker",
+ "model:single-provider",
+ ]
-def test_check_dependencies_returns_empty_when_no_redis_data(mocker: MockerFixture) -> None:
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.redis_client.get",
- return_value=None,
- )
- service = RagPipelineDslService(session=Mock())
- pipeline = Mock(id="p1", tenant_id="t1")
+def test_extract_dependencies_from_workflow_graph_ignores_malformed_nodes(
+ monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService
+) -> None:
+ monkeypatch.setattr(module.ToolNodeData, "model_validate", Mock(side_effect=ValueError("invalid tool")))
- result = service.check_dependencies(pipeline=pipeline)
+ dependencies = service._extract_dependencies_from_workflow_graph(
+ {"nodes": [{"data": {"type": BuiltinNodeTypes.TOOL}}]}
+ )
- assert result.leaked_dependencies == []
+ assert dependencies == []
-def test_check_dependencies_returns_leaked_deps_from_redis(mocker: MockerFixture) -> None:
+def test_check_dependencies_reads_redis_for_persisted_pipeline(
+ monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService, sqlite_session: Session
+) -> None:
from core.plugin.entities.plugin import PluginDependency, PluginDependencyType
from services.rag_pipeline.rag_pipeline_dsl_service import CheckDependenciesPendingData
- dep = PluginDependency(
+ pipeline = _pipeline(sqlite_session)
+ monkeypatch.setattr(module.redis_client, "get", Mock(return_value=None))
+ assert service.check_dependencies(pipeline=pipeline).leaked_dependencies == []
+ dependency = PluginDependency(
type=PluginDependencyType.Marketplace,
value=PluginDependency.Marketplace(marketplace_plugin_unique_identifier="test/plugin:0.1.0"),
)
- pending_data = CheckDependenciesPendingData(
- dependencies=[dep],
- pipeline_id="p1",
- )
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.redis_client.get",
- return_value=pending_data.model_dump_json(),
- )
- leaked = [dep]
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.get_leaked_dependencies",
- return_value=leaked,
- )
- service = RagPipelineDslService(session=Mock())
- pipeline = Mock(id="p1", tenant_id="t1")
-
- result = service.check_dependencies(pipeline=pipeline)
+ pending = CheckDependenciesPendingData(dependencies=[dependency], pipeline_id=pipeline.id)
+ monkeypatch.setattr(module.redis_client, "get", Mock(return_value=pending.model_dump_json()))
+ monkeypatch.setattr(module.DependenciesAnalysisService, "get_leaked_dependencies", Mock(return_value=[dependency]))
+ assert service.check_dependencies(pipeline=pipeline).leaked_dependencies == [dependency]
- assert result.leaked_dependencies == leaked
-
-# --- _extract_dependencies_from_model_config ---
-
-
-def test_extract_dependencies_from_model_config_extracts_model(mocker: MockerFixture) -> None:
- analyze_mock = mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.analyze_model_provider_dependency",
- return_value="langgenius/openai",
+ ("import_mode", "yaml_content", "yaml_url", "error"),
+ [
+ (ImportMode.YAML_URL.value, None, None, "yaml_url is required"),
+ (ImportMode.YAML_CONTENT.value, None, None, "yaml_content is required"),
+ (
+ ImportMode.YAML_CONTENT.value,
+ "- item",
+ None,
+ "content must be a mapping",
+ ),
+ (
+ ImportMode.YAML_CONTENT.value,
+ "version: 1\nkind: rag_pipeline",
+ None,
+ "Invalid version type",
+ ),
+ (
+ ImportMode.YAML_CONTENT.value,
+ "version: 0.1.0\nkind: rag_pipeline",
+ None,
+ "Missing rag_pipeline data",
+ ),
+ ],
+)
+def test_import_validation_errors(
+ service: RagPipelineDslService,
+ import_mode: str,
+ yaml_content: str | None,
+ yaml_url: str | None,
+ error: str,
+) -> None:
+ result = service.import_rag_pipeline(
+ account=_account(),
+ import_mode=import_mode,
+ yaml_content=yaml_content,
+ yaml_url=yaml_url,
)
- config = {"model": {"provider": "openai"}}
+ assert result.status == ImportStatus.FAILED
+ assert error in result.error
- result = RagPipelineDslService._extract_dependencies_from_model_config(config)
- assert "langgenius/openai" in result
- analyze_mock.assert_called_with("openai")
+def test_import_rejects_invalid_mode(service: RagPipelineDslService) -> None:
+ with pytest.raises(ValueError, match="Invalid import_mode"):
+ service.import_rag_pipeline(account=_account(), import_mode="invalid")
-def test_extract_dependencies_from_model_config_extracts_tools(mocker: MockerFixture) -> None:
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.analyze_model_provider_dependency",
- return_value="x",
- )
- analyze_tool_mock = mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.analyze_tool_dependency",
- return_value="langgenius/google",
+def test_import_url_boundary_failure(monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService) -> None:
+ monkeypatch.setattr(module.remote_fetcher, "make_request", Mock(side_effect=RuntimeError("network down")))
+ result = service.import_rag_pipeline(
+ account=_account(), import_mode=ImportMode.YAML_URL.value, yaml_url="https://example.com/pipeline.yml"
)
- config = {
- "model": {"provider": "openai"},
- "agent_mode": {"tools": [{"provider_id": "google"}]},
- }
-
- result = RagPipelineDslService._extract_dependencies_from_model_config(config)
-
- assert "langgenius/google" in result
- analyze_tool_mock.assert_called_with("google")
-
-
-def test_extract_dependencies_from_model_config_empty_config() -> None:
- result = RagPipelineDslService._extract_dependencies_from_model_config({})
-
- assert result == []
-
-
-# --- _extract_dependencies_from_workflow_graph ---
-
-
-def test_extract_dependencies_from_workflow_graph_ignores_unknown_types(mocker: MockerFixture) -> None:
- service = RagPipelineDslService(session=Mock())
- graph = {"nodes": [{"data": {"type": "some-unknown-type"}}]}
-
- result = service._extract_dependencies_from_workflow_graph(graph)
-
- assert result == []
-
-
-def test_extract_dependencies_from_workflow_graph_handles_empty_graph() -> None:
- service = RagPipelineDslService(session=Mock())
-
- result = service._extract_dependencies_from_workflow_graph({})
-
- assert result == []
-
-
-def test_extract_dependencies_from_workflow_graph_handles_malformed_node(mocker: MockerFixture) -> None:
- service = RagPipelineDslService(session=Mock())
- # Node with TOOL type but invalid data should be caught by exception handler
- from graphon.enums import BuiltinNodeTypes
-
- graph = {"nodes": [{"data": {"type": BuiltinNodeTypes.TOOL}}]}
-
- result = service._extract_dependencies_from_workflow_graph(graph)
-
- # Should not raise, error is caught internally
- assert isinstance(result, list)
-
-
-# --- export_rag_pipeline_dsl ---
-
-
-def test_export_rag_pipeline_dsl_raises_when_dataset_missing() -> None:
- pipeline = Mock()
- pipeline.retrieve_dataset.return_value = None
-
- service = RagPipelineDslService(session=Mock())
-
- with pytest.raises(ValueError, match="Missing dataset"):
- service.export_rag_pipeline_dsl(pipeline=pipeline)
-
-
-# --- import_rag_pipeline ---
+ assert result.status == ImportStatus.FAILED
+ assert "network down" in result.error
-def test_import_rag_pipeline_url_fetch_error(mocker: MockerFixture) -> None:
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.remote_fetcher.make_request",
- side_effect=Exception("fetch failed"),
- )
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
+def test_import_rejects_oversized_unicode_content_by_encoded_size(
+ monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService
+) -> None:
+ monkeypatch.setattr(module, "DSL_MAX_SIZE", 3)
result = service.import_rag_pipeline(
- account=account, import_mode="yaml-url", yaml_url="https://example.com/dsl.yml"
+ account=_account(),
+ import_mode=ImportMode.YAML_CONTENT.value,
+ yaml_content="你你",
)
assert result.status == ImportStatus.FAILED
- assert "fetch failed" in result.error
-
-
-def test_import_rag_pipeline_yaml_content_success(mocker: MockerFixture) -> None:
- yaml_content = """
-version: 0.1.0
-kind: rag_pipeline
-rag_pipeline:
- name: Test Pipeline
-workflow:
- graph:
- nodes:
- - data:
- type: knowledge-index
-"""
- pipeline = Mock()
- pipeline.name = "Test Pipeline"
- pipeline.description = "desc"
- pipeline.id = "p1"
- pipeline.is_published = False
- mocker.patch.object(RagPipelineDslService, "_create_or_update_pipeline", return_value=pipeline)
-
- config_mock = Mock()
- config_mock.indexing_technique = "high_quality"
- config_mock.embedding_model = "m"
- config_mock.embedding_model_provider = "p"
- config_mock.summary_index_setting = None
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeConfiguration.model_validate",
- return_value=config_mock,
- )
+ assert result.error == "File size exceeds the limit of 10MB"
- dataset_mock = Mock()
- dataset_mock.id = "d1"
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.Dataset", return_value=dataset_mock)
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.select", return_value=MagicMock())
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- session.scalars.return_value.all.return_value = []
- account = Mock(current_tenant_id="t1")
+ ("raw_content", "size_limit", "error"),
+ [
+ (b"\xff\xff", 1, "File size exceeds the limit of 10MB"),
+ (b"\xff", 2, "utf-8"),
+ (b"", 2, "Empty content from url"),
+ ],
+)
+def test_import_validates_url_bytes_before_parsing(
+ monkeypatch: pytest.MonkeyPatch,
+ service: RagPipelineDslService,
+ raw_content: bytes,
+ size_limit: int,
+ error: str,
+) -> None:
+ response = MagicMock()
+ response.content = raw_content
+ monkeypatch.setattr(module, "DSL_MAX_SIZE", size_limit)
+ monkeypatch.setattr(module.remote_fetcher, "make_request", Mock(return_value=response))
- result = service.import_rag_pipeline(account=account, import_mode="yaml-content", yaml_content=yaml_content)
+ result = service.import_rag_pipeline(
+ account=_account(),
+ import_mode=ImportMode.YAML_URL.value,
+ yaml_url="https://example.com/pipeline.yml",
+ )
- if result.status == ImportStatus.FAILED:
- print(f"DEBUG: {result.error}")
- assert result.status == ImportStatus.COMPLETED
- session.commit.assert_not_called()
- session.flush.assert_called()
+ assert result.status == ImportStatus.FAILED
+ assert error in result.error
-def test_import_rag_pipeline_flushes_new_collection_binding_without_commit(mocker: MockerFixture) -> None:
- yaml_content = """
-version: 0.1.0
-kind: rag_pipeline
+def test_import_supplies_default_version_and_kind(
+ monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService
+) -> None:
+ monkeypatch.setattr(module.KnowledgeConfiguration, "model_validate", Mock(return_value=_knowledge_configuration()))
+ yaml_content = f"""
rag_pipeline:
- name: Test Pipeline
+ name: Imported
workflow:
- graph:
- nodes:
- - data:
- type: knowledge-index
+ graph:
+ nodes:
+ - data:
+ type: {KNOWLEDGE_INDEX_NODE_TYPE}
"""
- pipeline = Mock(id="p1", description="desc", is_published=False)
- pipeline.name = "Test Pipeline"
- mocker.patch.object(RagPipelineDslService, "_create_or_update_pipeline", return_value=pipeline)
-
- config_mock = Mock()
- config_mock.indexing_technique = "high_quality"
- config_mock.embedding_model = "m"
- config_mock.embedding_model_provider = "p"
- config_mock.chunk_structure = "text_model"
- config_mock.retrieval_model.model_dump.return_value = {}
- config_mock.summary_index_setting = None
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeConfiguration.model_validate",
- return_value=config_mock,
- )
- dataset_mock = Mock(id="d1")
- binding_mock = Mock(id="b1")
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.Dataset", return_value=dataset_mock)
- binding_cls = mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DatasetCollectionBinding",
- return_value=binding_mock,
+ result = service.import_rag_pipeline(
+ account=_account(),
+ import_mode=ImportMode.YAML_CONTENT.value,
+ yaml_content=yaml_content,
)
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.select", return_value=MagicMock())
-
- session = cast(MagicMock, Mock())
- session.scalar.return_value = None
- session.scalars.return_value.all.return_value = []
- service = RagPipelineDslService(session=cast(Session, session))
- account = Mock(current_tenant_id="t1")
-
- result = service.import_rag_pipeline(account=account, import_mode="yaml-content", yaml_content=yaml_content)
assert result.status == ImportStatus.COMPLETED
- binding_cls.assert_called_once()
- assert dataset_mock.collection_binding_id == "b1"
- session.commit.assert_not_called()
- assert session.flush.call_count >= 2
-
-
-def test_import_rag_pipeline_pending_version(mocker: MockerFixture) -> None:
- yaml_content = "version: 1.0.0\nkind: rag_pipeline\nrag_pipeline: {name: x}"
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.redis_client.setex")
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1", id="u1")
-
- result = service.import_rag_pipeline(account=account, import_mode="yaml-content", yaml_content=yaml_content)
+ assert result.imported_dsl_version == module.CURRENT_DSL_VERSION
- assert result.status == ImportStatus.PENDING
- assert result.imported_dsl_version == "1.0.0"
+def test_confirm_import_rejects_non_serialized_pending_data(
+ monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService
+) -> None:
+ monkeypatch.setattr(module.redis_client, "get", Mock(return_value=object()))
-# --- confirm_import ---
+ result = service.confirm_import(import_id="import-1", account=_account())
+ assert result.status == ImportStatus.FAILED
+ assert result.error == "Invalid import information"
-def test_confirm_import_success(mocker: MockerFixture) -> None:
- from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelinePendingData
- yaml_content = """
-version: 0.1.0
-kind: rag_pipeline
-rag_pipeline:
- name: Test Pipeline
-workflow:
- graph:
- nodes:
- - data:
- type: knowledge-index
-"""
- pending = RagPipelinePendingData(import_mode="yaml-content", yaml_content=yaml_content, pipeline_id="p1")
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.redis_client.get",
- return_value=pending.model_dump_json(),
- )
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.redis_client.delete")
-
- pipeline = Mock()
- pipeline.id = "p1"
- pipeline.name = "Test Pipeline"
- pipeline.description = "desc"
- pipeline.retrieve_dataset.return_value = None
-
- mocker.patch.object(RagPipelineDslService, "_create_or_update_pipeline", return_value=pipeline)
-
- config_mock = Mock()
- config_mock.indexing_technique = "high_quality"
- config_mock.embedding_model = "m"
- config_mock.embedding_model_provider = "p"
- config_mock.chunk_structure = "text_model"
- config_mock.retrieval_model.model_dump.return_value = {}
- config_mock.summary_index_setting = None
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeConfiguration.model_validate",
- return_value=config_mock,
+def test_import_pending_version_stores_redis(monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService) -> None:
+ setex = Mock()
+ monkeypatch.setattr(module.redis_client, "setex", setex)
+ result = service.import_rag_pipeline(
+ account=_account(), import_mode=ImportMode.YAML_CONTENT.value, yaml_content=_valid_dsl(version="1.0.0")
)
+ assert result.status == ImportStatus.PENDING
+ setex.assert_called_once()
- dataset_mock = Mock()
- dataset_mock.id = "d1"
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.Dataset", return_value=dataset_mock)
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.DatasetCollectionBinding", return_value=Mock(id="b1"))
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.select", return_value=MagicMock())
-
- service = RagPipelineDslService(session=Mock())
- # Mocking self._session.scalar for the pipeline lookup
- service._session.scalar.return_value = pipeline
-
- account = Mock()
- account.id = "u1"
- account.current_tenant_id = "t1"
-
- result = service.confirm_import(account=account, import_id="imp-1")
+def test_import_creates_real_pipeline_dataset_binding_and_workflow_without_commit(
+ monkeypatch: pytest.MonkeyPatch,
+ service: RagPipelineDslService,
+ sqlite_session: Session,
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
+ monkeypatch.setattr(module.KnowledgeConfiguration, "model_validate", Mock(return_value=_knowledge_configuration()))
+ result = service.import_rag_pipeline(
+ account=_account(), import_mode=ImportMode.YAML_CONTENT.value, yaml_content=_valid_dsl()
+ )
assert result.status == ImportStatus.COMPLETED
- assert result.pipeline_id == "p1"
- assert result.dataset_id == "d1"
-
+ assert sqlite_session.in_transaction()
+ pipeline = sqlite_session.get(Pipeline, result.pipeline_id)
+ dataset = sqlite_session.get(Dataset, result.dataset_id)
+ assert pipeline is not None
+ assert dataset is not None
+ assert dataset.pipeline_id == pipeline.id
+ assert dataset.collection_binding_id is not None
+ assert sqlite_session.scalar(select(Workflow).where(Workflow.app_id == pipeline.id)) is not None
+ with sqlite_session_factory() as observer:
+ assert observer.get(Pipeline, pipeline.id) is None
+ assert observer.get(Dataset, dataset.id) is None
+
+ sqlite_session.commit()
+ with sqlite_session_factory() as observer:
+ assert observer.get(Pipeline, pipeline.id) is not None
+ assert observer.get(Dataset, dataset.id) is not None
+
+
+def test_import_pipeline_id_is_tenant_scoped(service: RagPipelineDslService, sqlite_session: Session) -> None:
+ foreign = _pipeline(sqlite_session, tenant_id="tenant-2")
+ result = service.import_rag_pipeline(
+ account=_account(tenant_id="tenant-1"),
+ import_mode=ImportMode.YAML_CONTENT.value,
+ yaml_content=_valid_dsl(),
+ pipeline_id=foreign.id,
+ )
+ assert result.status == ImportStatus.FAILED
+ assert result.error == "Pipeline not found"
-def test_confirm_import_flushes_new_collection_binding_without_commit(mocker: MockerFixture) -> None:
- from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelinePendingData
+def test_import_failure_leaves_rollback_to_caller(
+ service: RagPipelineDslService,
+ sqlite_session: Session,
+ sqlite_session_factory: sessionmaker[Session],
+) -> None:
yaml_content = """
version: 0.1.0
kind: rag_pipeline
rag_pipeline:
- name: Test Pipeline
+ name: Invalid
workflow:
- graph:
- nodes:
- - data:
- type: knowledge-index
+ graph:
+ nodes:
+ - data:
+ type: start
"""
- pending = RagPipelinePendingData(import_mode="yaml-content", yaml_content=yaml_content, pipeline_id="p1")
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.redis_client.get",
- return_value=pending.model_dump_json(),
- )
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.redis_client.delete")
-
- pipeline = Mock(id="p1", description="desc")
- pipeline.name = "Test Pipeline"
- pipeline.retrieve_dataset.return_value = None
- mocker.patch.object(RagPipelineDslService, "_create_or_update_pipeline", return_value=pipeline)
-
- config_mock = Mock()
- config_mock.indexing_technique = "high_quality"
- config_mock.embedding_model = "m"
- config_mock.embedding_model_provider = "p"
- config_mock.chunk_structure = "text_model"
- config_mock.retrieval_model.model_dump.return_value = {}
- config_mock.summary_index_setting = None
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeConfiguration.model_validate",
- return_value=config_mock,
- )
-
- dataset_mock = Mock(id="d1")
- binding_mock = Mock(id="b1")
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.Dataset", return_value=dataset_mock)
- binding_cls = mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DatasetCollectionBinding",
- return_value=binding_mock,
- )
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.select", return_value=MagicMock())
-
- session = cast(MagicMock, Mock())
- session.scalar.side_effect = [pipeline, None]
- service = RagPipelineDslService(session=cast(Session, session))
- account = Mock(id="u1", current_tenant_id="t1")
-
- result = service.confirm_import(account=account, import_id="imp-1")
-
- assert result.status == ImportStatus.COMPLETED
- binding_cls.assert_called_once()
- assert dataset_mock.collection_binding_id == "b1"
- session.commit.assert_not_called()
- assert session.flush.call_count >= 2
-
-
-# --- _extract_dependencies_from_workflow_graph all types ---
-
-
- "node_type",
- [
- BuiltinNodeTypes.TOOL,
- BuiltinNodeTypes.LLM,
- BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL,
- BuiltinNodeTypes.PARAMETER_EXTRACTOR,
- BuiltinNodeTypes.QUESTION_CLASSIFIER,
- ],
-)
-def test_extract_dependencies_from_workflow_graph_types(mocker, node_type) -> None:
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.analyze_tool_dependency",
- return_value="t1",
- )
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.analyze_model_provider_dependency",
- return_value="m1",
- )
-
- # Mock all potential node data classes
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.ToolNodeData.model_validate",
- return_value=Mock(provider_id="p1"),
- )
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.LLMNodeData.model_validate",
- return_value=Mock(model=Mock(provider="p1")),
- )
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeRetrievalNodeData.model_validate",
- return_value=Mock(
- retrieval_mode="single",
- single_retrieval_config=Mock(model=Mock(provider="p1")),
- ),
- )
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.ParameterExtractorNodeData.model_validate",
- return_value=Mock(model=Mock(provider="p1")),
- )
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.QuestionClassifierNodeData.model_validate",
- return_value=Mock(model=Mock(provider="p1")),
- )
-
- service = RagPipelineDslService(session=Mock())
- graph = {"nodes": [{"data": {"type": node_type}}]}
- result = service._extract_dependencies_from_workflow_graph(graph)
-
- assert len(result) > 0
-
-
-# --- _create_or_update_pipeline ---
-
-
-def test_create_or_update_pipeline_create_new(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- account = Mock(current_tenant_id="t1", id="u1")
- data = {
- "rag_pipeline": {"name": "New", "description": "desc"},
- "workflow": {"graph": {"nodes": []}},
- }
-
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.current_user", SimpleNamespace(id="u1"))
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.Workflow", return_value=Mock())
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.select", return_value=MagicMock())
- pipeline_cls = mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.Pipeline")
- pipeline_instance = pipeline_cls.return_value
- pipeline_instance.tenant_id = "t1"
- pipeline_instance.id = "p1"
- pipeline_instance.name = "P"
- pipeline_instance.is_published = False
- session.scalar.return_value = None
-
- result = service._create_or_update_pipeline(pipeline=None, data=data, account=account, dependencies=[])
-
- assert result == pipeline_instance
- session.add.assert_called()
- session.commit.assert_not_called()
- session.flush.assert_called()
-
-
-# --- export_rag_pipeline_dsl comprehensive ---
-
-
-def test_export_rag_pipeline_dsl_with_workflow(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- pipeline = Mock()
- pipeline.id = "p1"
- pipeline.tenant_id = "t1"
- pipeline.name = "P"
- pipeline.description = "d"
-
- dataset = Mock()
- dataset.id = "d1"
- dataset.name = "D"
- dataset.chunk_structure = "text_model"
- dataset.doc_form = "text_model"
- dataset.icon_info = {"icon": "i"}
- pipeline.retrieve_dataset.return_value = dataset
-
- workflow = Mock()
- workflow.app_id = "p1"
- workflow.graph_dict = {"nodes": []}
- workflow.environment_variables = []
- workflow.conversation_variables = []
- workflow.rag_pipeline_variables = []
- workflow.to_dict.return_value = {"graph": {"nodes": []}}
-
- session.scalar.return_value = workflow
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.generate_dependencies",
- return_value=[],
- )
-
- result_yaml = service.export_rag_pipeline_dsl(pipeline=pipeline)
- data = yaml.safe_load(result_yaml)
-
- assert data["kind"] == "rag_pipeline"
- assert data["rag_pipeline"]["name"] == "D"
- assert "workflow" in data
-
-
-# --- _extract_dependencies_from_workflow_graph more types ---
-
-
-def test_extract_dependencies_from_workflow_graph_datasource(mocker: MockerFixture) -> None:
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DatasourceNodeData.model_validate",
- return_value=Mock(provider_type="online", plugin_id="ds1"),
+ result = service.import_rag_pipeline(
+ account=_account(),
+ import_mode=ImportMode.YAML_CONTENT.value,
+ yaml_content=yaml_content,
)
- service = RagPipelineDslService(session=Mock())
- graph = {"nodes": [{"data": {"type": BuiltinNodeTypes.DATASOURCE}}]}
-
- result = service._extract_dependencies_from_workflow_graph(graph)
-
- assert "ds1" in result
-
-
-def test_import_rag_pipeline_raises_for_invalid_mode() -> None:
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
-
- with pytest.raises(ValueError, match="Invalid import_mode"):
- service.import_rag_pipeline(account=account, import_mode="invalid-mode")
-
-
-def test_import_rag_pipeline_yaml_url_requires_url() -> None:
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
-
- result = service.import_rag_pipeline(account=account, import_mode="yaml-url", yaml_url=None)
assert result.status == ImportStatus.FAILED
- assert "yaml_url is required" in result.error
-
-
-def test_import_rag_pipeline_yaml_content_requires_content() -> None:
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
-
- result = service.import_rag_pipeline(account=account, import_mode="yaml-content", yaml_content=None)
+ assert "Knowledge Index node" in result.error
+ assert sqlite_session.in_transaction()
+ assert sqlite_session.scalar(select(Pipeline).where(Pipeline.name == "Invalid")) is not None
+ with sqlite_session_factory() as observer:
+ assert observer.scalar(select(Pipeline).where(Pipeline.name == "Invalid")) is None
- assert result.status == ImportStatus.FAILED
- assert "yaml_content is required" in result.error
+ sqlite_session.rollback()
+ assert sqlite_session.scalar(select(Pipeline).where(Pipeline.name == "Invalid")) is None
-def test_import_rag_pipeline_rejects_oversized_yaml_content_before_parsing(
+def test_import_rejects_chunk_structure_change_for_published_pipeline(
monkeypatch: pytest.MonkeyPatch,
+ service: RagPipelineDslService,
+ sqlite_session: Session,
) -> None:
- monkeypatch.setattr("services.rag_pipeline.rag_pipeline_dsl_service.DSL_MAX_SIZE", 3)
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
+ pipeline = _pipeline(sqlite_session, published=True)
+ _dataset(sqlite_session, pipeline, chunk_structure="hierarchical_model")
+ _workflow(sqlite_session, pipeline)
+ monkeypatch.setattr(module.KnowledgeConfiguration, "model_validate", Mock(return_value=_knowledge_configuration()))
- result = service.import_rag_pipeline(account=account, import_mode="yaml-content", yaml_content="你你")
+ result = service.import_rag_pipeline(
+ account=_account(),
+ import_mode=ImportMode.YAML_CONTENT.value,
+ yaml_content=_valid_dsl(),
+ pipeline_id=pipeline.id,
+ )
assert result.status == ImportStatus.FAILED
- assert result.error == "File size exceeds the limit of 10MB"
-
+ assert result.error == "Chunk structure is not compatible with the published pipeline"
+ sqlite_session.rollback()
-def test_import_rag_pipeline_yaml_content_requires_mapping() -> None:
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
- result = service.import_rag_pipeline(account=account, import_mode="yaml-content", yaml_content="- one\n- two")
-
- assert result.status == ImportStatus.FAILED
- assert "content must be a mapping" in result.error
+def test_create_or_update_pipeline_flushes_caller_transaction_and_updates_existing(
+ service: RagPipelineDslService, sqlite_session: Session
+) -> None:
+ data: dict[str, Any] = {
+ "rag_pipeline": {"name": "New", "description": "description"},
+ "workflow": {"graph": {"nodes": [], "edges": []}},
+ }
+ created = service._create_or_update_pipeline(pipeline=None, data=data, account=_account())
+ assert created in sqlite_session
+ assert created.workflow_id is not None
+ assert sqlite_session.in_transaction()
+ sqlite_session.commit()
+
+ updated = service._create_or_update_pipeline(
+ pipeline=created,
+ data={
+ "rag_pipeline": {"name": "Updated", "description": "changed"},
+ "workflow": {"graph": {"nodes": [{"id": "node"}], "edges": []}},
+ },
+ account=_account(),
+ )
+ assert updated.id == created.id
+ assert updated.name == "Updated"
+ workflow = sqlite_session.get(Workflow, created.workflow_id)
+ assert workflow is not None
+ assert workflow.graph_dict["nodes"] == [{"id": "node"}]
-def test_import_rag_pipeline_rejects_oversized_yaml_content_by_bytes(
+def test_create_pipeline_flush_failure_is_rolled_back_by_caller(
+ service: RagPipelineDslService, sqlite_session: Session, sqlite_engine: Engine
+) -> None:
+ with _raise_on_workflow_insert(sqlite_engine), pytest.raises(RuntimeError, match="forced workflow INSERT"):
+ service._create_or_update_pipeline(
+ pipeline=None,
+ data={
+ "rag_pipeline": {"name": "Broken"},
+ "workflow": {"graph": {"nodes": [], "edges": []}},
+ },
+ account=_account(),
+ )
+ sqlite_session.rollback()
+ assert sqlite_session.scalar(select(Pipeline)) is None
+ assert sqlite_session.scalar(select(Workflow)) is None
+
+
+def test_confirm_import_updates_tenant_pipeline_and_dataset(
monkeypatch: pytest.MonkeyPatch,
+ service: RagPipelineDslService,
+ sqlite_session: Session,
+ sqlite_session_factory: sessionmaker[Session],
) -> None:
- monkeypatch.setattr("services.rag_pipeline.rag_pipeline_dsl_service.DSL_MAX_SIZE", 1)
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
-
- result = service.import_rag_pipeline(account=account, import_mode="yaml-content", yaml_content="é")
-
- assert result.status == ImportStatus.FAILED
- assert "10MB" in result.error
-
-
-def test_confirm_import_returns_failed_when_pending_data_is_invalid_type(mocker: MockerFixture) -> None:
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.redis_client.get", return_value=object())
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
-
- result = service.confirm_import(import_id="imp-1", account=account)
-
- assert result.status == ImportStatus.FAILED
- assert "Invalid import information" in result.error
-
-
-def test_append_workflow_export_data_filters_credentials(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- workflow = Mock()
- workflow.graph_dict = {"nodes": []}
- workflow.to_dict.return_value = {
- "graph": {
+ pipeline = _pipeline(sqlite_session)
+ dataset = _dataset(sqlite_session, pipeline)
+ _workflow(sqlite_session, pipeline)
+ pending = RagPipelinePendingData(
+ import_mode=ImportMode.YAML_CONTENT.value,
+ yaml_content=_valid_dsl(name="Confirmed"),
+ pipeline_id=pipeline.id,
+ )
+ monkeypatch.setattr(module.redis_client, "get", Mock(return_value=pending.model_dump_json()))
+ delete = Mock()
+ monkeypatch.setattr(module.redis_client, "delete", delete)
+ monkeypatch.setattr(module.KnowledgeConfiguration, "model_validate", Mock(return_value=_knowledge_configuration()))
+ result = service.confirm_import(import_id="import-1", account=_account())
+ assert result.status == ImportStatus.COMPLETED
+ assert result.pipeline_id == pipeline.id
+ assert result.dataset_id == dataset.id
+ persisted_pipeline = sqlite_session.get(Pipeline, pipeline.id)
+ assert persisted_pipeline is not None
+ assert persisted_pipeline.name == "Confirmed"
+ assert sqlite_session.in_transaction()
+ with sqlite_session_factory() as observer:
+ observed_pipeline = observer.get(Pipeline, pipeline.id)
+ assert observed_pipeline is not None
+ assert observed_pipeline.name == "Pipeline"
+
+ sqlite_session.commit()
+ with sqlite_session_factory() as observer:
+ observed_pipeline = observer.get(Pipeline, pipeline.id)
+ assert observed_pipeline is not None
+ assert observed_pipeline.name == "Confirmed"
+
+ delete.assert_called_once()
+
+
+def test_export_reads_real_dataset_and_workflow_and_filters_credentials(
+ monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService, sqlite_session: Session
+) -> None:
+ pipeline = _pipeline(sqlite_session)
+ with pytest.raises(ValueError, match="Missing dataset"):
+ service.export_rag_pipeline_dsl(pipeline)
+ _dataset(sqlite_session, pipeline)
+ with pytest.raises(ValueError, match="Missing draft workflow"):
+ service.export_rag_pipeline_dsl(pipeline)
+ _workflow(
+ sqlite_session,
+ pipeline,
+ graph={
"nodes": [
{
"data": {
"type": BuiltinNodeTypes.TOOL,
"credential_id": "secret",
+ "provider_id": "provider",
+ }
+ },
+ {
+ "data": {
+ "type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL,
+ "dataset_ids": ["dataset-1"],
}
},
{
"data": {
"type": BuiltinNodeTypes.AGENT,
- "agent_parameters": {"tools": {"value": [{"credential_id": "secret-agent"}]}},
+ "agent_parameters": {
+ "tools": {"value": [{"credential_id": "agent-secret"}]},
+ },
}
},
- ]
- }
- }
- session.scalar.return_value = workflow
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.generate_dependencies",
- return_value=[],
+ ],
+ "edges": [],
+ },
)
- export_data: dict[str, Any] = {}
- pipeline = Mock(id="p1", tenant_id="t1")
+ monkeypatch.setattr(module.DependenciesAnalysisService, "generate_dependencies", Mock(return_value=[]))
+ exported = yaml.safe_load(service.export_rag_pipeline_dsl(pipeline))
+ assert exported["kind"] == "rag_pipeline"
+ nodes = exported["workflow"]["graph"]["nodes"]
+ assert "credential_id" not in nodes[0]["data"]
+ assert service.decrypt_dataset_id(nodes[1]["data"]["dataset_ids"][0], pipeline.tenant_id) == "dataset-1"
+ assert "credential_id" not in nodes[2]["data"]["agent_parameters"]["tools"]["value"][0]
- service._append_workflow_export_data(export_data=export_data, pipeline=pipeline, include_secret=False)
- nodes = export_data["workflow"]["graph"]["nodes"]
- assert "credential_id" not in nodes[0]["data"]
- assert "credential_id" not in nodes[1]["data"]["agent_parameters"]["tools"]["value"][0]
-
-
-def test_create_rag_pipeline_dataset_raises_when_name_conflicts(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- session.scalar.return_value = Mock()
- create_entity = RagPipelineDatasetCreateEntity(
- name="Existing Name",
- description="",
- icon_info=IconInfo(icon="book"),
- permission="only_me",
- yaml_content="x",
- )
-
- with pytest.raises(ValueError, match="already exists"):
- service.create_rag_pipeline_dataset("tenant-1", create_entity)
-
-
-def test_create_rag_pipeline_dataset_generates_name_when_missing(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- session.scalar.return_value = None
- session.scalars.return_value.all.return_value = [Mock(name="Untitled")]
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.generate_incremental_name", return_value="Untitled 2")
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.current_user", Mock(id="u1", current_tenant_id="t1"))
- mocker.patch.object(
- service,
- "import_rag_pipeline",
- return_value=SimpleNamespace(
- id="imp-1",
- dataset_id="d1",
- pipeline_id="p1",
- status=ImportStatus.COMPLETED,
- imported_dsl_version="0.1.0",
- current_dsl_version="0.1.0",
- error="",
- ),
- )
- create_entity = RagPipelineDatasetCreateEntity(
- name="",
- description="",
- icon_info=IconInfo(icon="book"),
- permission="only_me",
- yaml_content="x",
- )
-
- result = service.create_rag_pipeline_dataset("tenant-1", create_entity)
-
- assert create_entity.name == "Untitled 2"
- assert result["status"] == ImportStatus.COMPLETED
-
-
-def test_append_workflow_export_data_encrypts_knowledge_retrieval_dataset_ids(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- workflow = Mock()
- workflow.graph_dict = {"nodes": []}
- workflow.to_dict.return_value = {
- "graph": {
+def test_export_preserves_tool_and_agent_credentials_when_requested(
+ monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService, sqlite_session: Session
+) -> None:
+ pipeline = _pipeline(sqlite_session)
+ _dataset(sqlite_session, pipeline)
+ _workflow(
+ sqlite_session,
+ pipeline,
+ graph={
"nodes": [
{
"data": {
- "type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL,
- "dataset_ids": ["d1", "d2"],
- }
- }
- ]
- }
- }
- session.scalar.return_value = workflow
- mocker.patch.object(service, "encrypt_dataset_id", side_effect=lambda dataset_id, tenant_id: f"enc-{dataset_id}")
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.generate_dependencies",
- return_value=[],
- )
- export_data: dict[str, Any] = {}
- pipeline = Mock(id="p1", tenant_id="t1")
-
- service._append_workflow_export_data(export_data=export_data, pipeline=pipeline, include_secret=False)
-
- ids = export_data["workflow"]["graph"]["nodes"][0]["data"]["dataset_ids"]
- assert ids == ["enc-d1", "enc-d2"]
-
-
-def test_confirm_import_updates_existing_dataset(mocker: MockerFixture) -> None:
- from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelinePendingData
-
- yaml_content = (
- "version: 0.1.0\n"
- "kind: rag_pipeline\n"
- "rag_pipeline: {name: x}\n"
- "workflow: {graph: {nodes: [{data: {type: knowledge-index}}]}}"
- )
- pending = RagPipelinePendingData(import_mode="yaml-content", yaml_content=yaml_content, pipeline_id="p1")
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.redis_client.get",
- return_value=pending.model_dump_json(),
- )
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.redis_client.delete")
- pipeline = Mock(id="p1", name="P", description="D")
- dataset = Mock(id="d1")
- pipeline.retrieve_dataset.return_value = dataset
- mocker.patch.object(RagPipelineDslService, "_create_or_update_pipeline", return_value=pipeline)
- config_mock = Mock()
- config_mock.indexing_technique = "economy"
- config_mock.keyword_number = 3
- config_mock.retrieval_model.model_dump.return_value = {"top_k": 3}
- config_mock.chunk_structure = "text_model"
- config_mock.summary_index_setting = None
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeConfiguration.model_validate",
- return_value=config_mock,
- )
- service = RagPipelineDslService(session=Mock())
- service._session.scalar.return_value = pipeline
- account = Mock(id="u1", current_tenant_id="t1")
-
- result = service.confirm_import(import_id="imp-1", account=account)
-
- assert result.status == ImportStatus.COMPLETED
- assert dataset.indexing_technique == "economy"
-
-
-def test_import_rag_pipeline_yaml_url_handles_empty_content_after_github_rewrite(mocker: MockerFixture) -> None:
- response = Mock()
- response.raise_for_status.return_value = None
- response.content = b""
- get_mock = mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.remote_fetcher.make_request",
- return_value=response,
- )
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
-
- result = service.import_rag_pipeline(
- account=account,
- import_mode="yaml-url",
- yaml_url="https://github.com/langgenius/dify/blob/main/pipeline.yml",
- )
-
- assert result.status == ImportStatus.FAILED
- assert "Empty content from url" in result.error
- called_url = get_mock.call_args.args[1]
- assert "raw.githubusercontent.com" in called_url
-
-
-def test_create_or_update_pipeline_decrypts_knowledge_retrieval_dataset_ids(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- account = Mock(id="u1", current_tenant_id="t1")
- pipeline = Mock(id="p1", tenant_id="t1", name="N", description="D")
- data = {
- "rag_pipeline": {"name": "N2", "description": "D2"},
- "workflow": {
- "graph": {
- "nodes": [
- {
- "data": {
- "type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL,
- "dataset_ids": ["enc-1", "enc-2"],
- }
+ "type": BuiltinNodeTypes.TOOL,
+ "credential_id": "tool-secret",
+ "provider_id": "provider",
}
- ]
- }
- },
- }
- draft_workflow = Mock(id="wf1")
- session.scalar.return_value = draft_workflow
- mocker.patch.object(service, "decrypt_dataset_id", side_effect=["d1", None])
-
- result = service._create_or_update_pipeline(pipeline=pipeline, data=data, account=account)
-
- assert result is pipeline
- assert data["workflow"]["graph"]["nodes"][0]["data"]["dataset_ids"] == ["d1"]
- assert draft_workflow.graph is not None
-
-
-def test_create_or_update_pipeline_creates_draft_when_missing(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- account = Mock(id="u1", current_tenant_id="t1")
- pipeline = Mock(id="p1", tenant_id="t1", name="N", description="D")
- data = {"rag_pipeline": {"name": "N2", "description": "D2"}, "workflow": {"graph": {"nodes": []}}}
- session.scalar.return_value = None
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.select", return_value=MagicMock())
- workflow_cls = mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.Workflow")
- workflow_cls.return_value.id = "wf-new"
-
- service._create_or_update_pipeline(pipeline=pipeline, data=data, account=account)
-
- assert pipeline.workflow_id == "wf-new"
-
-
-def test_import_rag_pipeline_url_size_exceeds_limit(mocker: MockerFixture) -> None:
- response = Mock()
- response.raise_for_status.return_value = None
- response.content = b"x" * (10 * 1024 * 1024 + 1)
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.remote_fetcher.make_request", return_value=response)
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
-
- result = service.import_rag_pipeline(
- account=account,
- import_mode="yaml-url",
- yaml_url="https://example.com/pipeline.yaml",
- )
-
- assert result.status == ImportStatus.FAILED
- assert "10MB" in result.error
-
-
-def test_import_rag_pipeline_rejects_oversized_yaml_url_bytes_before_decode(
- mocker: MockerFixture,
- monkeypatch: pytest.MonkeyPatch,
-) -> None:
- monkeypatch.setattr("services.rag_pipeline.rag_pipeline_dsl_service.DSL_MAX_SIZE", 1)
- response = Mock()
- response.raise_for_status.return_value = None
- response.content = b"\xff\xff"
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.remote_fetcher.make_request", return_value=response)
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
-
- result = service.import_rag_pipeline(
- account=account,
- import_mode="yaml-url",
- yaml_url="https://example.com/pipeline.yaml",
- )
-
- assert result.status == ImportStatus.FAILED
- assert "10MB" in result.error
-
-
-def test_import_rag_pipeline_returns_decode_error_for_invalid_yaml_url_bytes(mocker: MockerFixture) -> None:
- response = Mock()
- response.raise_for_status.return_value = None
- response.content = b"\xff"
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.remote_fetcher.make_request", return_value=response)
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
-
- result = service.import_rag_pipeline(
- account=account,
- import_mode="yaml-url",
- yaml_url="https://example.com/pipeline.yaml",
- )
-
- assert result.status == ImportStatus.FAILED
- assert "utf-8" in result.error
-
-
-def test_import_rag_pipeline_fails_when_rag_pipeline_data_missing() -> None:
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
- result = service.import_rag_pipeline(
- account=account,
- import_mode="yaml-content",
- yaml_content="version: 0.1.0\nkind: rag_pipeline\nworkflow: {}",
- )
-
- assert result.status == ImportStatus.FAILED
- assert "Missing rag_pipeline data" in result.error
-
-
-def test_import_rag_pipeline_fails_when_pipeline_id_not_found() -> None:
- session = cast(MagicMock, Mock())
- session.scalar.return_value = None
- service = RagPipelineDslService(session=cast(Session, session))
- account = Mock(current_tenant_id="t1")
-
- result = service.import_rag_pipeline(
- account=account,
- import_mode="yaml-content",
- yaml_content="version: 0.1.0\nkind: rag_pipeline\nrag_pipeline: {name: x}\nworkflow: {}",
- pipeline_id="missing-pipeline",
- )
-
- assert result.status == ImportStatus.FAILED
- assert "Pipeline not found" in result.error
-
-
-def test_import_rag_pipeline_fails_for_non_string_version_type() -> None:
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1")
-
- result = service.import_rag_pipeline(
- account=account,
- import_mode="yaml-content",
- yaml_content="version: 1\nkind: rag_pipeline\nrag_pipeline: {name: x}\nworkflow: {}",
- )
-
- assert result.status == ImportStatus.FAILED
- assert "Invalid version type" in result.error
-
-
-def test_append_workflow_export_data_raises_when_draft_workflow_missing() -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- session.scalar.return_value = None
-
- with pytest.raises(ValueError, match="Missing draft workflow configuration"):
- service._append_workflow_export_data(export_data={}, pipeline=Mock(tenant_id="t1"), include_secret=False)
-
-
-def test_append_workflow_export_data_keeps_secret_fields_when_include_secret_true(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- workflow = Mock()
- workflow.graph_dict = {"nodes": []}
- workflow.to_dict.return_value = {
- "graph": {
- "nodes": [
- {"data": {"type": BuiltinNodeTypes.TOOL, "credential_id": "tool-secret"}},
+ },
{
"data": {
"type": BuiltinNodeTypes.AGENT,
- "agent_parameters": {"tools": {"value": [{"credential_id": "agent-secret"}]}},
+ "agent_parameters": {
+ "tools": {"value": [{"credential_id": "agent-secret"}]},
+ },
}
},
- ]
- }
- }
- session.scalar.return_value = workflow
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.generate_dependencies",
- return_value=[],
- )
-
- export_data: dict[str, object] = {}
- service._append_workflow_export_data(export_data=export_data, pipeline=Mock(tenant_id="t1"), include_secret=True)
-
- workflow_data = cast(dict[str, object], export_data["workflow"])
- graph = cast(dict[str, object], workflow_data["graph"])
- nodes = cast(list[dict[str, object]], graph["nodes"])
- node0_data = cast(dict[str, object], nodes[0]["data"])
- node1_data = cast(dict[str, object], nodes[1]["data"])
- agent_parameters = cast(dict[str, object], node1_data["agent_parameters"])
- tools = cast(dict[str, object], agent_parameters["tools"])
- tool_values = cast(list[dict[str, object]], tools["value"])
- assert node0_data["credential_id"] == "tool-secret"
- assert tool_values[0]["credential_id"] == "agent-secret"
-
-
-def test_extract_dependencies_from_workflow_graph_skips_local_file_datasource(mocker: MockerFixture) -> None:
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DatasourceNodeData.model_validate",
- return_value=Mock(provider_type="local_file", plugin_id="plugin-x"),
- )
- service = RagPipelineDslService(session=Mock())
-
- result = service._extract_dependencies_from_workflow_graph(
- {"nodes": [{"data": {"type": BuiltinNodeTypes.DATASOURCE}}]}
- )
-
- assert result == []
-
-
-def test_extract_dependencies_from_workflow_graph_knowledge_index_reranking(mocker: MockerFixture) -> None:
- analyze = mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.analyze_model_provider_dependency",
- side_effect=lambda provider: f"dep:{provider}",
- )
- knowledge = Mock()
- knowledge.indexing_technique = "high_quality"
- knowledge.embedding_model_provider = "embed-provider"
- knowledge.retrieval_model.reranking_mode = "reranking_model"
- knowledge.retrieval_model.reranking_enable = True
- knowledge.retrieval_model.reranking_model.reranking_provider_name = "rerank-provider"
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeConfiguration.model_validate",
- return_value=knowledge,
- )
- service = RagPipelineDslService(session=Mock())
-
- result = service._extract_dependencies_from_workflow_graph(
- {"nodes": [{"data": {"type": KNOWLEDGE_INDEX_NODE_TYPE}}]}
- )
-
- assert result == ["dep:embed-provider", "dep:rerank-provider"]
- assert analyze.call_count == 2
-
-
-def test_extract_dependencies_from_workflow_graph_multiple_retrieval_weighted_score(mocker: MockerFixture) -> None:
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.analyze_model_provider_dependency",
- return_value="dep:weighted",
- )
- retrieval = Mock()
- retrieval.retrieval_mode = "multiple"
- retrieval.multiple_retrieval_config.reranking_mode = "weighted_score"
- retrieval.multiple_retrieval_config.weights.vector_setting.embedding_provider_name = "emb-provider"
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeRetrievalNodeData.model_validate",
- return_value=retrieval,
- )
- service = RagPipelineDslService(session=Mock())
-
- result = service._extract_dependencies_from_workflow_graph(
- {"nodes": [{"data": {"type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL}}]}
- )
-
- assert result == ["dep:weighted"]
-
-
-def test_extract_dependencies_from_workflow_graph_multiple_retrieval_reranking_model(mocker: MockerFixture) -> None:
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.analyze_model_provider_dependency",
- return_value="dep:rerank",
- )
- retrieval = Mock()
- retrieval.retrieval_mode = "multiple"
- retrieval.multiple_retrieval_config.reranking_mode = "reranking_model"
- retrieval.multiple_retrieval_config.reranking_model.provider = "rerank-provider"
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeRetrievalNodeData.model_validate",
- return_value=retrieval,
- )
- service = RagPipelineDslService(session=Mock())
-
- result = service._extract_dependencies_from_workflow_graph(
- {"nodes": [{"data": {"type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL}}]}
- )
-
- assert result == ["dep:rerank"]
-
-
-def test_extract_dependencies_from_model_config_includes_dataset_reranking_and_tools(mocker: MockerFixture) -> None:
- model_analyze = mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.analyze_model_provider_dependency",
- side_effect=["dep:model", "dep:rerank"],
- )
- tool_analyze = mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.analyze_tool_dependency",
- return_value="dep:tool",
- )
- config = {
- "model": {"provider": "openai"},
- "dataset_configs": {
- "datasets": {
- "datasets": [
- {
- "reranking_model": {
- "reranking_provider_name": {"provider": "cohere"},
- }
- }
- ]
- }
+ ],
+ "edges": [],
},
- "agent_mode": {"tools": [{"provider_id": "google"}]},
- }
-
- deps = RagPipelineDslService._extract_dependencies_from_model_config(config)
-
- assert deps == ["dep:model", "dep:rerank", "dep:tool"]
- assert model_analyze.call_count == 2
- tool_analyze.assert_called_once_with("google")
-
-
-def test_check_version_compatibility_hits_major_older_branch(mocker: MockerFixture) -> None:
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.CURRENT_DSL_VERSION", "1.0.0")
-
- status = check_version_compatibility("0.9.0", rag_pipeline_dsl_service.CURRENT_DSL_VERSION)
-
- assert status == ImportStatus.PENDING
-
-
-def test_import_rag_pipeline_sets_default_version_and_kind(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- account = Mock(current_tenant_id="t1")
- pipeline = Mock(id="p1", name="P", description="D", is_published=False)
- mocker.patch.object(service, "_create_or_update_pipeline", return_value=pipeline)
- config = Mock()
- config.indexing_technique = "economy"
- config.keyword_number = 2
- config.retrieval_model.model_dump.return_value = {}
- config.summary_index_setting = None
- config.chunk_structure = "text_model"
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeConfiguration.model_validate",
- return_value=config,
- )
- dataset = Mock(id="d1")
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.Dataset", return_value=dataset)
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.select", return_value=MagicMock())
- session.scalars.return_value.all.return_value = []
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.generate_incremental_name", return_value="P")
-
- result = service.import_rag_pipeline(
- account=account,
- import_mode="yaml-content",
- yaml_content="rag_pipeline: {name: x}\nworkflow: {graph: {nodes: [{data: {type: knowledge-index}}]}}",
)
+ monkeypatch.setattr(module.DependenciesAnalysisService, "generate_dependencies", Mock(return_value=[]))
- assert result.status == ImportStatus.COMPLETED
- assert result.imported_dsl_version == "0.1.0"
-
-
-def test_import_rag_pipeline_creates_pending_for_dependencies(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- account = Mock(current_tenant_id="t1")
- setex = mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.redis_client.setex")
- yaml_content = """
-version: 1.0.0
-kind: rag_pipeline
-rag_pipeline: {name: x}
-dependencies:
- - type: marketplace
- value:
- marketplace_plugin_unique_identifier: langgenius/example:0.1.0
-workflow: {graph: {nodes: []}}
-"""
+ exported = yaml.safe_load(service.export_rag_pipeline_dsl(pipeline, include_secret=True))
- result = service.import_rag_pipeline(account=account, import_mode="yaml-content", yaml_content=yaml_content)
+ nodes = exported["workflow"]["graph"]["nodes"]
+ assert nodes[0]["data"]["credential_id"] == "tool-secret"
+ assert nodes[1]["data"]["agent_parameters"]["tools"]["value"][0]["credential_id"] == "agent-secret"
- assert result.status == ImportStatus.PENDING
- setex.assert_called_once()
-
-
-def test_confirm_import_returns_failed_when_pending_pipeline_missing(mocker: MockerFixture) -> None:
- from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelinePendingData
-
- pending = RagPipelinePendingData(import_mode="yaml-content", yaml_content="version: 0.1.0", pipeline_id="p1")
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.redis_client.get", return_value=pending.model_dump_json()
- )
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- session.scalar.return_value = None
- mocker.patch.object(RagPipelineDslService, "_create_or_update_pipeline", side_effect=ValueError("pipeline missing"))
-
- result = service.confirm_import(import_id="imp-1", account=Mock(current_tenant_id="t1"))
-
- assert result.status == ImportStatus.FAILED
-
-
-def test_append_workflow_export_data_skips_empty_node_data(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- workflow = Mock()
- workflow.graph_dict = {"nodes": []}
- workflow.to_dict.return_value = {"graph": {"nodes": [{"data": {}}, {}]}}
- session.scalar.return_value = workflow
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.generate_dependencies",
- return_value=[],
- )
- export_data = {}
-
- service._append_workflow_export_data(export_data=export_data, pipeline=Mock(tenant_id="t1"), include_secret=False)
-
- assert "workflow" in export_data
-
-
-def test_extract_dependencies_from_workflow_graph_multiple_config_none(mocker: MockerFixture) -> None:
- retrieval = Mock()
- retrieval.retrieval_mode = "multiple"
- retrieval.multiple_retrieval_config = None
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeRetrievalNodeData.model_validate",
- return_value=retrieval,
- )
- service = RagPipelineDslService(session=Mock())
-
- result = service._extract_dependencies_from_workflow_graph(
- {"nodes": [{"data": {"type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL}}]}
- )
-
- assert result == []
-
-
-def test_extract_dependencies_from_workflow_graph_single_config_none(mocker: MockerFixture) -> None:
- retrieval = Mock()
- retrieval.retrieval_mode = "single"
- retrieval.single_retrieval_config = None
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeRetrievalNodeData.model_validate",
- return_value=retrieval,
- )
- service = RagPipelineDslService(session=Mock())
-
- result = service._extract_dependencies_from_workflow_graph(
- {"nodes": [{"data": {"type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL}}]}
- )
-
- assert result == []
-
-
-def test_create_or_update_pipeline_raises_when_workflow_missing() -> None:
- service = RagPipelineDslService(session=Mock())
- account = Mock(current_tenant_id="t1", id="u1")
-
- with pytest.raises(ValueError, match="Missing workflow data for rag pipeline"):
- service._create_or_update_pipeline(pipeline=None, data={"rag_pipeline": {"name": "x"}}, account=account)
-
-
-def test_import_rag_pipeline_with_pipeline_id_uses_existing_dataset(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- existing_dataset = Mock(id="d1", chunk_structure="text_model")
- existing_pipeline = Mock(id="p1", name="P", description="D", is_published=False)
- existing_pipeline.retrieve_dataset.return_value = existing_dataset
- session.scalar.return_value = existing_pipeline
- mocker.patch.object(service, "_create_or_update_pipeline", return_value=existing_pipeline)
- config = Mock()
- config.indexing_technique = "economy"
- config.keyword_number = 3
- config.chunk_structure = "text_model"
- config.summary_index_setting = {"enabled": True}
- config.retrieval_model.model_dump.return_value = {"top_k": 3}
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeConfiguration.model_validate", return_value=config
- )
-
- yaml_content = (
- "version: 0.1.0\n"
- "kind: rag_pipeline\n"
- "rag_pipeline: {name: x}\n"
- "workflow: {graph: {nodes: [{data: {type: knowledge-index}}]}}"
- )
-
- result = service.import_rag_pipeline(
- account=Mock(id="u1", current_tenant_id="t1"),
- import_mode="yaml-content",
- yaml_content=yaml_content,
- pipeline_id="p1",
- )
- assert result.status == ImportStatus.COMPLETED
- assert result.dataset_id == "d1"
-
-
-def test_import_rag_pipeline_raises_for_chunk_structure_mismatch_on_published(mocker: MockerFixture) -> None:
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- existing_dataset = Mock(id="d1", chunk_structure="hierarchical_model")
- existing_pipeline = Mock(id="p1", name="P", description="D", is_published=True)
- existing_pipeline.retrieve_dataset.return_value = existing_dataset
- session.scalar.return_value = existing_pipeline
- mocker.patch.object(service, "_create_or_update_pipeline", return_value=existing_pipeline)
- config = Mock()
- config.chunk_structure = "text_model"
- config.indexing_technique = "economy"
- config.keyword_number = 3
- config.summary_index_setting = None
- config.retrieval_model.model_dump.return_value = {}
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeConfiguration.model_validate", return_value=config
- )
-
- yaml_content = (
- "version: 0.1.0\n"
- "kind: rag_pipeline\n"
- "rag_pipeline: {name: x}\n"
- "workflow: {graph: {nodes: [{data: {type: knowledge-index}}]}}"
- )
-
- result = service.import_rag_pipeline(
- account=Mock(id="u1", current_tenant_id="t1"),
- import_mode="yaml-content",
- yaml_content=yaml_content,
- pipeline_id="p1",
- )
-
- assert result.status == ImportStatus.FAILED
- assert "Chunk structure is not compatible" in result.error
-
-
-def test_import_rag_pipeline_fails_when_no_knowledge_index_node(mocker: MockerFixture) -> None:
- service = RagPipelineDslService(session=Mock())
- pipeline = Mock(id="p1", name="P", description="D", is_published=False)
- mocker.patch.object(service, "_create_or_update_pipeline", return_value=pipeline)
-
- yaml_content = (
- "version: 0.1.0\n"
- "kind: rag_pipeline\n"
- "rag_pipeline: {name: x}\n"
- "workflow: {graph: {nodes: [{data: {type: start}}]}}"
- )
-
- result = service.import_rag_pipeline(
- account=Mock(id="u1", current_tenant_id="t1"),
- import_mode="yaml-content",
- yaml_content=yaml_content,
- )
-
- assert result.status == ImportStatus.FAILED
- assert "Knowledge Index node" in result.error
-
-
-def test_confirm_import_fails_when_no_knowledge_index_node(mocker: MockerFixture) -> None:
- from services.rag_pipeline.rag_pipeline_dsl_service import RagPipelinePendingData
-
- yaml_content = (
- "version: 0.1.0\n"
- "kind: rag_pipeline\n"
- "rag_pipeline: {name: x}\n"
- "workflow: {graph: {nodes: [{data: {type: start}}]}}"
- )
-
- pending = RagPipelinePendingData(
- import_mode="yaml-content",
- yaml_content=yaml_content,
- pipeline_id=None,
- )
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.redis_client.get", return_value=pending.model_dump_json()
- )
- service = RagPipelineDslService(session=Mock())
- pipeline = Mock(id="p1", name="P", description="D")
- pipeline.retrieve_dataset.return_value = None
- mocker.patch.object(service, "_create_or_update_pipeline", return_value=pipeline)
-
- result = service.confirm_import(import_id="imp-1", account=Mock(id="u1", current_tenant_id="t1"))
-
- assert result.status == ImportStatus.FAILED
- assert "Knowledge Index node" in result.error
-
-
-def test_create_or_update_pipeline_saves_dependencies_to_redis(mocker: MockerFixture) -> None:
- from core.plugin.entities.plugin import PluginDependency, PluginDependencyType
-
- session = cast(MagicMock, Mock())
- service = RagPipelineDslService(session=cast(Session, session))
- account = Mock(id="u1", current_tenant_id="t1")
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.current_user", SimpleNamespace(id="u1"))
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.Workflow", return_value=Mock(id="wf-1"))
- mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.select", return_value=MagicMock())
- pipeline_cls = mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.Pipeline")
- pipeline = pipeline_cls.return_value
- pipeline.tenant_id = "t1"
- pipeline.id = "p1"
- session.scalar.return_value = None
- setex = mocker.patch("services.rag_pipeline.rag_pipeline_dsl_service.redis_client.setex")
- dependency = PluginDependency(
- type=PluginDependencyType.Marketplace,
- value=PluginDependency.Marketplace(marketplace_plugin_unique_identifier="langgenius/example:0.1.0"),
- )
-
- service._create_or_update_pipeline(
- pipeline=None,
- data={"rag_pipeline": {"name": "x"}, "workflow": {"graph": {"nodes": []}}},
- account=account,
- dependencies=[dependency],
- )
-
- setex.assert_called_once()
-
-
-def test_extract_dependencies_from_workflow_graph_knowledge_index_without_embedding_provider(
- mocker: MockerFixture,
+def test_create_dataset_name_is_tenant_scoped_and_ignores_template_rows(
+ monkeypatch: pytest.MonkeyPatch, service: RagPipelineDslService, sqlite_session: Session
) -> None:
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.DependenciesAnalysisService.analyze_model_provider_dependency",
- return_value="dep",
- )
- knowledge = Mock()
- knowledge.indexing_technique = "high_quality"
- knowledge.embedding_model_provider = None
- knowledge.retrieval_model.reranking_mode = "reranking_model"
- knowledge.retrieval_model.reranking_enable = False
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeConfiguration.model_validate", return_value=knowledge
- )
- service = RagPipelineDslService(session=Mock())
-
- result = service._extract_dependencies_from_workflow_graph(
- {"nodes": [{"data": {"type": KNOWLEDGE_INDEX_NODE_TYPE}}]}
- )
-
- assert result == []
-
-
-def test_extract_dependencies_from_workflow_graph_multiple_reranking_without_model(mocker: MockerFixture) -> None:
- retrieval = Mock()
- retrieval.retrieval_mode = "multiple"
- retrieval.multiple_retrieval_config.reranking_mode = "reranking_model"
- retrieval.multiple_retrieval_config.reranking_model = None
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeRetrievalNodeData.model_validate",
- return_value=retrieval,
- )
- service = RagPipelineDslService(session=Mock())
-
- result = service._extract_dependencies_from_workflow_graph(
- {"nodes": [{"data": {"type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL}}]}
- )
-
- assert result == []
-
-
-def test_extract_dependencies_from_workflow_graph_multiple_weighted_without_weights(mocker: MockerFixture) -> None:
- retrieval = Mock()
- retrieval.retrieval_mode = "multiple"
- retrieval.multiple_retrieval_config.reranking_mode = "weighted_score"
- retrieval.multiple_retrieval_config.weights = None
- mocker.patch(
- "services.rag_pipeline.rag_pipeline_dsl_service.KnowledgeRetrievalNodeData.model_validate",
- return_value=retrieval,
- )
- service = RagPipelineDslService(session=Mock())
-
- result = service._extract_dependencies_from_workflow_graph(
- {"nodes": [{"data": {"type": BuiltinNodeTypes.KNOWLEDGE_RETRIEVAL}}]}
+ foreign_pipeline = _pipeline(sqlite_session, tenant_id="tenant-2")
+ _dataset(sqlite_session, foreign_pipeline, name="Shared")
+ template = PipelineCustomizedTemplate(
+ tenant_id="tenant-1",
+ name="Shared",
+ description="template",
+ chunk_structure="text_model",
+ icon={},
+ position=1,
+ yaml_content=_valid_dsl(),
+ install_count=0,
+ language="en-US",
+ created_by="account-1",
+ )
+ sqlite_session.add(template)
+ sqlite_session.commit()
+ imported = Mock(
+ id="import-1",
+ dataset_id="dataset-1",
+ pipeline_id="pipeline-1",
+ status=ImportStatus.COMPLETED,
+ imported_dsl_version="0.1.0",
+ current_dsl_version="0.1.0",
+ error="",
+ )
+ monkeypatch.setattr(service, "import_rag_pipeline", Mock(return_value=imported))
+ monkeypatch.setattr(module, "current_user", _account())
+ result = service.create_rag_pipeline_dataset(
+ "tenant-1",
+ RagPipelineDatasetCreateEntity(
+ name="Shared",
+ description="description",
+ yaml_content=_valid_dsl(),
+ icon_info=IconInfo(icon="📙"),
+ permission="only_me",
+ ),
)
+ assert result["dataset_id"] == "dataset-1"
- assert result == []
+ local_pipeline = _pipeline(sqlite_session, tenant_id="tenant-1")
+ _dataset(sqlite_session, local_pipeline, name="Local")
+ with pytest.raises(ValueError, match="already exists"):
+ service.create_rag_pipeline_dataset(
+ "tenant-1",
+ RagPipelineDatasetCreateEntity(
+ name="Local",
+ description="description",
+ yaml_content=_valid_dsl(),
+ icon_info=IconInfo(icon="📙"),
+ permission="only_me",
+ ),
+ )