Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

test: migrate RAG pipeline sessions and ORM models to SQLite (#40511)

Asuka Minatoce1aed8df58e
diff --git a/api/core/app/apps/pipeline/pipeline_generator.py b/api/core/app/apps/pipeline/pipeline_generator.py
index f6cae29ce35..c3e1be5a95b 100644
--- a/api/core/app/apps/pipeline/pipeline_generator.py
+++ b/api/core/app/apps/pipeline/pipeline_generator.py
@@ -654,8 +654,6 @@ class PipelineGenerator(BaseAppGenerator):
except Exception as e:
logger.exception("Unknown Error when generating")
queue_manager.publish_error(e, PublishFrom.APPLICATION_MANAGER)
- finally:
- db.session.close()
def _handle_response(
self,
diff --git a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_config_manager.py b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_config_manager.py
index 6c1ee20ffbc..a24aec4cb92 100644
--- a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_config_manager.py
+++ b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_config_manager.py
@@ -1,16 +1,31 @@
+import json
from types import SimpleNamespace
-from unittest.mock import MagicMock
from pytest_mock import MockerFixture
import core.app.apps.pipeline.pipeline_config_manager as module
from core.app.apps.pipeline.pipeline_config_manager import PipelineConfigManager
+from models.dataset import Pipeline
from models.model import AppMode
+from models.workflow import Workflow, WorkflowType
def test_get_pipeline_config(mocker: MockerFixture):
- pipeline = MagicMock(tenant_id="tenant", id="pipe1")
- workflow = MagicMock(id="wf1")
+ pipeline = Pipeline(tenant_id="tenant", name="Pipeline", description="")
+ pipeline.id = "pipe1"
+ workflow = Workflow.new(
+ tenant_id="tenant",
+ app_id="pipe1",
+ type=WorkflowType.RAG_PIPELINE.value,
+ version=Workflow.VERSION_DRAFT,
+ graph=json.dumps({"nodes": [], "edges": []}),
+ features="{}",
+ created_by="user",
+ environment_variables=[],
+ conversation_variables=[],
+ rag_pipeline_variables=[],
+ )
+ workflow.id = "wf1"
mocker.patch.object(
module.WorkflowVariablesConfigManager,
diff --git a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py
index f1edcd1a1b1..6234297fc3e 100644
--- a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py
+++ b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_generator.py
@@ -1,4 +1,5 @@
import contextlib
+import json
from types import SimpleNamespace
from unittest.mock import MagicMock, PropertyMock
@@ -6,13 +7,13 @@ import pytest
from pytest_mock import MockerFixture
from sqlalchemy import select
from sqlalchemy.engine import Engine
-from sqlalchemy.orm import Session, scoped_session, sessionmaker
+from sqlalchemy.orm import Session
import core.app.apps.pipeline.pipeline_generator as module
from core.app.apps.exc import GenerateTaskStoppedError
from core.app.entities.app_invoke_entities import InvokeFrom
from core.datasource.entities.datasource_entities import DatasourceProviderType
-from models.dataset import Document, DocumentPipelineExecutionLog
+from models.dataset import Dataset, Document, DocumentPipelineExecutionLog, Pipeline
from models.enums import DataSourceType, EndUserType
from models.model import EndUser
from models.workflow import Workflow, WorkflowType
@@ -49,28 +50,51 @@ def generator(mocker: MockerFixture, sqlite_engine: Engine):
def _build_pipeline_dataset():
- return SimpleNamespace(
+ return Dataset(
id=DATASET_ID,
+ tenant_id=TENANT_ID,
name="dataset",
description="desc",
+ created_by=USER_ID,
+ pipeline_id=PIPELINE_ID,
chunk_structure="text_model",
built_in_field_enabled=True,
- tenant_id=TENANT_ID,
)
def _build_pipeline():
- pipeline = MagicMock(tenant_id=TENANT_ID, id=PIPELINE_ID)
- pipeline.retrieve_dataset.return_value = _build_pipeline_dataset()
+ pipeline = Pipeline(tenant_id=TENANT_ID, name="Pipeline", description="desc")
+ pipeline.id = PIPELINE_ID
+ pipeline.workflow_id = WORKFLOW_ID
return pipeline
-def _build_workflow():
- return MagicMock(id=WORKFLOW_ID, graph_dict={"nodes": [], "edges": []}, tenant_id=TENANT_ID)
+def _build_workflow(*, graph: dict | None = None):
+ workflow = Workflow.new(
+ tenant_id=TENANT_ID,
+ app_id=PIPELINE_ID,
+ type=WorkflowType.RAG_PIPELINE.value,
+ version=Workflow.VERSION_DRAFT,
+ graph=json.dumps(graph if graph is not None else {"nodes": [], "edges": []}),
+ features="{}",
+ created_by=USER_ID,
+ environment_variables=[],
+ conversation_variables=[],
+ rag_pipeline_variables=[],
+ )
+ workflow.id = WORKFLOW_ID
+ return workflow
def _build_user():
- return SimpleNamespace(id=USER_ID, name="User", session_id="session")
+ return EndUser(
+ id=USER_ID,
+ tenant_id=TENANT_ID,
+ app_id=PIPELINE_ID,
+ type=EndUserType.BROWSER,
+ name="User",
+ session_id="session",
+ )
def _build_args():
@@ -86,10 +110,19 @@ def _patch_sqlite_engine(mocker: MockerFixture, sqlite_engine: Engine) -> None:
mocker.patch.object(type(module.db), "engine", new_callable=PropertyMock, return_value=sqlite_engine)
-def _patch_db_session(mocker: MockerFixture, sqlite_session_factory: sessionmaker[Session]) -> scoped_session[Session]:
- session_proxy = scoped_session(sqlite_session_factory)
- mocker.patch.object(module.db, "session", session_proxy)
- return session_proxy
+def _persist_pipeline_scope(
+ session: Session,
+ *,
+ pipeline: Pipeline | None = None,
+ dataset: Dataset | None = None,
+ workflow: Workflow | None = None,
+) -> tuple[Pipeline, Dataset, Workflow]:
+ pipeline = pipeline or _build_pipeline()
+ dataset = dataset or _build_pipeline_dataset()
+ workflow = workflow or _build_workflow()
+ session.add_all([pipeline, dataset, workflow])
+ session.commit()
+ return pipeline, dataset, workflow
def _persist_worker_records(session: Session) -> None:
@@ -122,7 +155,6 @@ def _dummy_preserve(*args, **kwargs):
def test_generate_dataset_missing(generator, sqlite_session: Session):
pipeline = _build_pipeline()
- pipeline.retrieve_dataset.return_value = None
with pytest.raises(ValueError):
generator.generate(
@@ -137,8 +169,7 @@ def test_generate_dataset_missing(generator, sqlite_session: Session):
def test_generate_debugger_calls_generate(generator, mocker: MockerFixture, sqlite_session: Session):
- pipeline = _build_pipeline()
- workflow = _build_workflow()
+ pipeline, _, workflow = _persist_pipeline_scope(sqlite_session)
mocker.patch.object(
generator,
@@ -181,8 +212,7 @@ def test_generate_debugger_calls_generate(generator, mocker: MockerFixture, sqli
def test_generate_published_pipeline_creates_documents_and_delay(
generator, mocker: MockerFixture, sqlite_session: Session
):
- pipeline = _build_pipeline()
- workflow = _build_workflow()
+ pipeline, _, workflow = _persist_pipeline_scope(sqlite_session)
datasource_info_list = [{"name": "file1"}, {"name": "file2"}]
@@ -245,8 +275,7 @@ def test_generate_published_pipeline_creates_documents_and_delay(
def test_generate_published_pipeline_rejects_when_document_creation_limits_exceeded(
generator, mocker: MockerFixture, sqlite_session: Session
):
- pipeline = _build_pipeline()
- workflow = _build_workflow()
+ pipeline, _, workflow = _persist_pipeline_scope(sqlite_session)
datasource_info_list = [{"name": "file1"}, {"name": "file2"}]
mocker.patch.object(
@@ -283,8 +312,7 @@ def test_generate_published_pipeline_rejects_when_document_creation_limits_excee
def test_generate_is_retry_calls_generate(generator, mocker: MockerFixture, sqlite_session: Session):
- pipeline = _build_pipeline()
- workflow = _build_workflow()
+ pipeline, _, workflow = _persist_pipeline_scope(sqlite_session)
mocker.patch.object(
generator,
@@ -336,14 +364,12 @@ def test_generate_worker_handles_errors(
mocker: MockerFixture,
sqlite_session: Session,
sqlite_engine: Engine,
- sqlite_session_factory: sessionmaker[Session],
):
flask_app = MagicMock()
flask_app.app_context.return_value = contextlib.nullcontext()
mocker.patch.object(module, "preserve_flask_contexts", _dummy_preserve)
_persist_worker_records(sqlite_session)
_patch_sqlite_engine(mocker, sqlite_engine)
- _patch_db_session(mocker, sqlite_session_factory)
application_generate_entity = FakeRagPipelineGenerateEntity(
app_config=SimpleNamespace(tenant_id=TENANT_ID, app_id=PIPELINE_ID, workflow_id=WORKFLOW_ID),
@@ -374,14 +400,12 @@ def test_generate_worker_sets_system_user_id_for_external_call(
mocker: MockerFixture,
sqlite_session: Session,
sqlite_engine: Engine,
- sqlite_session_factory: sessionmaker[Session],
):
flask_app = MagicMock()
flask_app.app_context.return_value = contextlib.nullcontext()
mocker.patch.object(module, "preserve_flask_contexts", _dummy_preserve)
_persist_worker_records(sqlite_session)
_patch_sqlite_engine(mocker, sqlite_engine)
- _patch_db_session(mocker, sqlite_session_factory)
application_generate_entity = FakeRagPipelineGenerateEntity(
app_config=SimpleNamespace(tenant_id=TENANT_ID, app_id=PIPELINE_ID, workflow_id=WORKFLOW_ID),
@@ -503,7 +527,6 @@ def test_single_iteration_generate_validates_inputs(generator, sqlite_session: S
def test_single_iteration_generate_dataset_required(generator, sqlite_session: Session):
pipeline = _build_pipeline()
- pipeline.retrieve_dataset.return_value = None
with pytest.raises(ValueError):
generator.single_iteration_generate(
@@ -520,9 +543,8 @@ def test_single_iteration_generate_success(
generator,
mocker: MockerFixture,
sqlite_session: Session,
- sqlite_session_factory: sessionmaker[Session],
):
- pipeline = _build_pipeline()
+ pipeline, _, workflow = _persist_pipeline_scope(sqlite_session)
mocker.patch.object(
module.PipelineConfigManager,
@@ -539,8 +561,6 @@ def test_single_iteration_generate_success(
"create_workflow_node_execution_repository",
return_value=MagicMock(),
)
- _patch_db_session(mocker, sqlite_session_factory)
-
mocker.patch.object(module, "WorkflowDraftVariableService", return_value=MagicMock())
mocker.patch.object(module, "DraftVarLoader", return_value=MagicMock())
@@ -548,7 +568,7 @@ def test_single_iteration_generate_success(
result = generator.single_iteration_generate(
pipeline,
- _build_workflow(),
+ workflow,
"node",
_build_user(),
{"inputs": {"a": 1}},
@@ -563,9 +583,8 @@ def test_single_loop_generate_success(
generator,
mocker: MockerFixture,
sqlite_session: Session,
- sqlite_session_factory: sessionmaker[Session],
):
- pipeline = _build_pipeline()
+ pipeline, _, workflow = _persist_pipeline_scope(sqlite_session)
mocker.patch.object(
module.PipelineConfigManager,
@@ -582,8 +601,6 @@ def test_single_loop_generate_success(
"create_workflow_node_execution_repository",
return_value=MagicMock(),
)
- _patch_db_session(mocker, sqlite_session_factory)
-
mocker.patch.object(module, "WorkflowDraftVariableService", return_value=MagicMock())
mocker.patch.object(module, "DraftVarLoader", return_value=MagicMock())
@@ -591,7 +608,7 @@ def test_single_loop_generate_success(
result = generator.single_loop_generate(
pipeline,
- _build_workflow(),
+ workflow,
"node",
_build_user(),
{"inputs": {"a": 1}},
@@ -622,12 +639,7 @@ def test_handle_response_value_error_triggers_generate_task_stopped(generator, m
)
-def test_build_document_sets_metadata_for_builtin_fields(generator, mocker: MockerFixture):
- class DummyDocument(SimpleNamespace):
- pass
-
- mocker.patch.object(module, "Document", side_effect=lambda **kwargs: DummyDocument(**kwargs))
-
+def test_build_document_sets_metadata_for_builtin_fields(generator):
document = generator._build_document(
tenant_id="tenant",
dataset_id="ds",
@@ -693,7 +705,7 @@ def test_format_datasource_info_list_non_online_drive(generator):
def test_format_datasource_info_list_missing_node_data(generator):
- workflow = MagicMock(graph_dict={"nodes": []})
+ workflow = _build_workflow()
with pytest.raises(ValueError):
generator._format_datasource_info_list(
@@ -707,8 +719,8 @@ def test_format_datasource_info_list_missing_node_data(generator):
def test_format_datasource_info_list_online_drive_folder(generator, mocker: MockerFixture):
- workflow = MagicMock(
- graph_dict={
+ workflow = _build_workflow(
+ graph={
"nodes": [
{
"id": "start",
@@ -720,7 +732,7 @@ def test_format_datasource_info_list_online_drive_folder(generator, mocker: Mock
},
}
]
- }
+ },
)
runtime = MagicMock()
diff --git a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_runner.py b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_runner.py
index f81620d6606..3e847f65d73 100644
--- a/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_runner.py
+++ b/api/tests/unit_tests/core/app/apps/pipeline/test_pipeline_runner.py
@@ -1,10 +1,3 @@
-"""
-Unit tests for PipelineRunner behavior.
-Asserts correct event handling, error propagation, and user invocation logic.
-Primary collaborators: PipelineRunner, InvokeFrom, GraphRunFailedEvent, UserFrom, and mocked dependencies.
-Cross-references: core.app.apps.pipeline.pipeline_runner, core.app.entities.app_invoke_entities.
-"""
-
"""Unit tests for PipelineRunner behavior.
This module validates core control-flow outcomes for
@@ -18,16 +11,100 @@ Primary collaborators include ``PipelineRunner``,
``UserFrom``, and patched DB/runtime dependencies used by the runner.
"""
+import json
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from pytest_mock import MockerFixture
+from sqlalchemy import event
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
import core.app.apps.pipeline.pipeline_runner as module
from core.app.apps.pipeline.pipeline_runner import PipelineRunner
from core.app.entities.app_invoke_entities import InvokeFrom, UserFrom
from graphon.graph_events import GraphRunFailedEvent
+from models.dataset import Dataset, Document, Pipeline
+from models.enums import DataSourceType, DocumentCreatedFrom, EndUserType
+from models.model import EndUser
+from models.workflow import Workflow, WorkflowType
+
+
+def _pipeline(*, tenant_id: str = "tenant", pipeline_id: str = "pipe") -> Pipeline:
+ pipeline = Pipeline(tenant_id=tenant_id, name="Pipeline", description="")
+ pipeline.id = pipeline_id
+ pipeline.workflow_id = "wf"
+ return pipeline
+
+
+def _dataset(*, tenant_id: str = "tenant", dataset_id: str = "ds", pipeline_id: str = "pipe") -> Dataset:
+ return Dataset(
+ id=dataset_id,
+ tenant_id=tenant_id,
+ name="Dataset",
+ description="",
+ created_by="user",
+ pipeline_id=pipeline_id,
+ )
+
+
+def _workflow(*, tenant_id: str = "tenant", pipeline_id: str = "pipe", graph: dict | None = None) -> Workflow:
+ return Workflow.new(
+ tenant_id=tenant_id,
+ app_id=pipeline_id,
+ type=WorkflowType.RAG_PIPELINE.value,
+ version="v1",
+ graph=json.dumps(graph if graph is not None else {"nodes": [], "edges": []}),
+ features="{}",
+ created_by="user",
+ environment_variables=[],
+ conversation_variables=[],
+ rag_pipeline_variables=[],
+ )
+
+
+def _end_user() -> EndUser:
+ return EndUser(
+ id="user",
+ tenant_id="tenant",
+ app_id="pipe",
+ type=EndUserType.BROWSER,
+ name="User",
+ session_id="sess",
+ )
+
+
+def _document(*, document_id: str = "doc", dataset_id: str = "ds", tenant_id: str = "tenant") -> Document:
+ return Document(
+ id=document_id,
+ tenant_id=tenant_id,
+ dataset_id=dataset_id,
+ position=1,
+ data_source_type=DataSourceType.UPLOAD_FILE,
+ batch="batch",
+ name="Document",
+ created_from=DocumentCreatedFrom.API,
+ created_by="user",
+ )
+
+
+def _persist_scope(
+ session: Session,
+ *,
+ pipeline: Pipeline | None = None,
+ dataset: Dataset | None = None,
+ workflow: Workflow | None = None,
+ end_user: EndUser | None = None,
+ documents: tuple[Document, ...] = (),
+) -> tuple[Pipeline, Dataset, Workflow]:
+ pipeline = pipeline or _pipeline()
+ dataset = dataset or _dataset(tenant_id=pipeline.tenant_id, pipeline_id=pipeline.id)
+ workflow = workflow or _workflow(tenant_id=pipeline.tenant_id, pipeline_id=pipeline.id)
+ workflow.id = "wf"
+ session.add_all([pipeline, dataset, workflow, *(documents or ()), *([end_user] if end_user else [])])
+ session.commit()
+ return pipeline, dataset, workflow
def _build_app_generate_entity() -> SimpleNamespace:
@@ -53,38 +130,12 @@ def _build_app_generate_entity() -> SimpleNamespace:
)
-def _patch_create_session(mocker: MockerFixture, session: MagicMock, *, events: list[str] | None = None):
- """Patch create_session() to yield ``session`` inside its ``with`` body and ``begin()`` block.
-
- The runner now obtains short-lived sessions via ``create_session()`` instead of the
- Flask scoped ``db.session``, so tests patch the module-level ``create_session`` and
- hand back a context manager that yields the mock session.
- """
- session_context = MagicMock()
-
- def enter_session():
- if events is not None:
- events.append("session_enter")
- return session
-
- def exit_session(*args):
- if events is not None:
- events.append("session_exit")
- return False
-
- session_context.__enter__.side_effect = enter_session
- session_context.__exit__.side_effect = exit_session
- session.begin.return_value.__enter__.return_value = session
- session.begin.return_value.__exit__.return_value = False
- return mocker.patch.object(module, "create_session", return_value=session_context)
-
-
@pytest.fixture
def runner():
app_generate_entity = _build_app_generate_entity()
queue_manager = MagicMock()
variable_loader = MagicMock()
- workflow = MagicMock()
+ workflow = _workflow()
workflow_execution_repository = MagicMock()
workflow_node_execution_repository = MagicMock()
@@ -103,105 +154,91 @@ def test_get_app_id(runner):
assert runner._get_app_id() == "pipe"
-def test_get_workflow_returns_workflow(runner):
- pipeline = MagicMock(tenant_id="tenant", id="pipe")
- workflow = MagicMock(id="wf")
-
- session = MagicMock()
- session.scalar.return_value = workflow
+def test_get_workflow_returns_workflow(runner, sqlite_session: Session):
+ pipeline, _, workflow = _persist_scope(sqlite_session)
- result = runner.get_workflow(session=session, pipeline=pipeline, workflow_id="wf")
+ result = runner.get_workflow(session=sqlite_session, pipeline=pipeline, workflow_id="wf")
assert result == workflow
def test_init_rag_pipeline_graph_invalid_config(mocker, runner):
- workflow = MagicMock(id="wf", tenant_id="tenant", graph_dict={})
+ workflow = _workflow(graph={})
with pytest.raises(ValueError):
runner._init_rag_pipeline_graph(workflow=workflow, graph_runtime_state=MagicMock())
- workflow.graph_dict = {"nodes": "bad", "edges": []}
+ workflow.graph = json.dumps({"nodes": "bad", "edges": []})
with pytest.raises(ValueError):
runner._init_rag_pipeline_graph(workflow=workflow, graph_runtime_state=MagicMock())
- workflow.graph_dict = {"nodes": [], "edges": "bad"}
+ workflow.graph = json.dumps({"nodes": [], "edges": "bad"})
with pytest.raises(ValueError):
runner._init_rag_pipeline_graph(workflow=workflow, graph_runtime_state=MagicMock())
def test_init_rag_pipeline_graph_not_found(mocker, runner):
- workflow = MagicMock(id="wf", tenant_id="tenant", graph_dict={"nodes": [], "edges": []})
+ workflow = _workflow()
mocker.patch.object(module.Graph, "init", return_value=None)
with pytest.raises(ValueError):
runner._init_rag_pipeline_graph(workflow=workflow, graph_runtime_state=MagicMock())
-def test_update_document_status_on_failure(mocker, runner):
- document = MagicMock()
- document_ref = MagicMock()
-
- session = MagicMock()
- _patch_create_session(mocker, session)
- get_document_by_ref = mocker.patch.object(
- module.DatasetRefService,
- "get_document_by_ref",
- return_value=document,
- )
+def test_update_document_status_on_failure(runner, sqlite_session: Session):
+ document = _document()
+ _, dataset, _ = _persist_scope(sqlite_session, documents=(document,))
+ dataset_ref = module.DatasetRefService.create_dataset_ref(dataset)
+ document_ref = module.DatasetRefService.create_document_ref_from_id(dataset_ref, document.id)
event = GraphRunFailedEvent(error="boom")
runner._update_document_status(event, document_ref)
- get_document_by_ref.assert_called_once_with(document_ref, session=session)
- assert document.indexing_status == "error"
- assert document.error == "boom"
- session.add.assert_called_once_with(document)
- session.begin.assert_called_once()
- session.begin.return_value.__enter__.assert_called_once()
- session.begin.return_value.__exit__.assert_called_once()
-
-
-def test_update_document_status_skips_when_document_not_found(mocker, runner):
- document_ref = MagicMock()
- session = MagicMock()
- _patch_create_session(mocker, session)
- get_document_by_ref = mocker.patch.object(
- module.DatasetRefService,
- "get_document_by_ref",
- return_value=None,
- )
+ sqlite_session.expire_all()
+ updated = sqlite_session.get(Document, document.id)
+ assert updated is not None
+ assert updated.indexing_status == "error"
+ assert updated.error == "boom"
+
+
+def test_update_document_status_skips_when_document_not_found(runner, sqlite_session: Session):
+ _, dataset, _ = _persist_scope(sqlite_session)
+ dataset_ref = module.DatasetRefService.create_dataset_ref(dataset)
+ document_ref = module.DatasetRefService.create_document_ref_from_id(dataset_ref, "missing")
runner._update_document_status(GraphRunFailedEvent(error="boom"), document_ref)
- get_document_by_ref.assert_called_once_with(document_ref, session=session)
- session.add.assert_not_called()
+ assert sqlite_session.get(Document, "missing") is None
-def test_update_document_status_skips_without_document_ref(mocker, runner):
- create_session = mocker.patch.object(module, "create_session")
+def test_update_document_status_skips_without_document_ref(runner, sqlite_engine: Engine):
+ checkouts = 0
- runner._update_document_status(GraphRunFailedEvent(error="boom"), None)
+ def record_checkout(*_args) -> None:
+ nonlocal checkouts
+ checkouts += 1
- create_session.assert_not_called()
+ event.listen(sqlite_engine, "checkout", record_checkout)
+ try:
+ runner._update_document_status(GraphRunFailedEvent(error="boom"), None)
+ finally:
+ event.remove(sqlite_engine, "checkout", record_checkout)
+ assert checkouts == 0
-def test_run_pipeline_not_found(mocker: MockerFixture):
+
+def test_run_pipeline_not_found():
app_generate_entity = _build_app_generate_entity()
app_generate_entity.invoke_from = InvokeFrom.WEB_APP
app_generate_entity.single_iteration_run = None
app_generate_entity.single_loop_run = None
- session = MagicMock()
- session.get.side_effect = [None, None]
- _patch_create_session(mocker, session)
-
runner = PipelineRunner(
application_generate_entity=app_generate_entity,
queue_manager=MagicMock(),
variable_loader=MagicMock(),
- workflow=MagicMock(),
+ workflow=_workflow(),
system_user_id="sys",
workflow_execution_repository=MagicMock(),
workflow_node_execution_repository=MagicMock(),
@@ -211,177 +248,116 @@ def test_run_pipeline_not_found(mocker: MockerFixture):
runner.run()
-def test_run_pipeline_from_other_tenant_is_not_found(mocker: MockerFixture, runner: PipelineRunner):
- pipeline = MagicMock(id="pipe", tenant_id="other-tenant")
- session = MagicMock()
- session.get.side_effect = [None, pipeline]
- _patch_create_session(mocker, session)
+def test_run_pipeline_from_other_tenant_is_not_found(runner: PipelineRunner, sqlite_session: Session):
+ pipeline = _pipeline(tenant_id="other-tenant")
+ sqlite_session.add(pipeline)
+ sqlite_session.commit()
with pytest.raises(ValueError, match="Pipeline not found"):
runner.run()
- pipeline.retrieve_dataset.assert_not_called()
-
@pytest.mark.parametrize(
"dataset",
[
pytest.param(None, id="missing"),
- pytest.param(SimpleNamespace(id="ds", tenant_id="other-tenant"), id="other-tenant"),
- pytest.param(SimpleNamespace(id="other-dataset", tenant_id="tenant"), id="other-dataset"),
+ pytest.param(_dataset(tenant_id="other-tenant"), id="other-tenant"),
+ pytest.param(_dataset(dataset_id="other-dataset"), id="other-dataset"),
],
)
def test_run_rejects_unowned_pipeline_dataset(
- mocker: MockerFixture,
runner: PipelineRunner,
- dataset: SimpleNamespace | None,
+ dataset: Dataset | None,
+ sqlite_session: Session,
):
- pipeline = MagicMock(id="pipe", tenant_id="tenant")
- pipeline.retrieve_dataset.return_value = dataset
- session = MagicMock()
- session.get.side_effect = [None, pipeline]
- _patch_create_session(mocker, session)
+ pipeline = _pipeline()
+ sqlite_session.add(pipeline)
+ if dataset is not None:
+ sqlite_session.add(dataset)
+ sqlite_session.commit()
runner.get_workflow = MagicMock()
with pytest.raises(ValueError, match="Pipeline dataset not found"):
runner.run()
- pipeline.retrieve_dataset.assert_called_once_with(session)
runner.get_workflow.assert_not_called()
def test_run_rejects_document_outside_pipeline_dataset_after_async_boundary(
- mocker: MockerFixture,
runner: PipelineRunner,
+ sqlite_session: Session,
):
runner.application_generate_entity.document_id = "foreign-doc"
runner.application_generate_entity.original_document_id = "foreign-doc"
- pipeline = MagicMock(id="pipe", tenant_id="tenant")
- pipeline.retrieve_dataset.return_value = SimpleNamespace(id="ds", tenant_id="tenant")
- session = MagicMock()
- session.get.side_effect = [None, pipeline]
- _patch_create_session(mocker, session)
+ _persist_scope(sqlite_session)
runner.get_workflow = MagicMock()
- get_document_by_ref = mocker.patch.object(
- module.DatasetRefService,
- "get_document_by_ref",
- return_value=None,
- )
with pytest.raises(ValueError, match="Pipeline document not found"):
runner.run()
- document_ref = get_document_by_ref.call_args.args[0]
- assert document_ref.dataset.tenant_id == "tenant"
- assert document_ref.dataset.dataset_id == "ds"
- assert document_ref.document_id == "foreign-doc"
- get_document_by_ref.assert_called_once_with(document_ref, session=session)
runner.get_workflow.assert_not_called()
def test_run_rejects_original_document_outside_pipeline_dataset_after_async_boundary(
- mocker: MockerFixture,
runner: PipelineRunner,
+ sqlite_session: Session,
):
runner.application_generate_entity.document_id = "doc"
runner.application_generate_entity.original_document_id = "foreign-doc"
- pipeline = MagicMock(id="pipe", tenant_id="tenant")
- pipeline.retrieve_dataset.return_value = SimpleNamespace(id="ds", tenant_id="tenant")
- session = MagicMock()
- session.get.side_effect = [None, pipeline]
- _patch_create_session(mocker, session)
+ _persist_scope(sqlite_session, documents=(_document(),))
runner.get_workflow = MagicMock()
- get_document_by_ref = mocker.patch.object(
- module.DatasetRefService,
- "get_document_by_ref",
- side_effect=[SimpleNamespace(id="doc"), None],
- )
with pytest.raises(ValueError, match="Pipeline original document not found"):
runner.run()
- document_refs = [call.args[0] for call in get_document_by_ref.call_args_list]
- assert [document_ref.document_id for document_ref in document_refs] == ["doc", "foreign-doc"]
- for document_ref in document_refs:
- assert document_ref.dataset.tenant_id == "tenant"
- assert document_ref.dataset.dataset_id == "ds"
runner.get_workflow.assert_not_called()
-def test_run_workflow_not_initialized(mocker: MockerFixture):
+def test_run_workflow_not_initialized(sqlite_session: Session):
app_generate_entity = _build_app_generate_entity()
- pipeline = MagicMock(id="pipe", tenant_id="tenant")
- pipeline.retrieve_dataset.return_value = SimpleNamespace(id="ds", tenant_id="tenant")
-
- session = MagicMock()
- session.get.side_effect = [None, pipeline]
- _patch_create_session(mocker, session)
+ pipeline = _pipeline()
+ dataset = _dataset()
+ document = _document()
+ sqlite_session.add_all([pipeline, dataset, document])
+ sqlite_session.commit()
runner = PipelineRunner(
application_generate_entity=app_generate_entity,
queue_manager=MagicMock(),
variable_loader=MagicMock(),
- workflow=MagicMock(),
+ workflow=_workflow(),
system_user_id="sys",
workflow_execution_repository=MagicMock(),
workflow_node_execution_repository=MagicMock(),
)
- runner.get_workflow = MagicMock(return_value=None)
-
with pytest.raises(ValueError):
runner.run()
-def test_run_single_iteration_path(mocker: MockerFixture):
+def test_run_single_iteration_path(mocker: MockerFixture, sqlite_session: Session):
app_generate_entity = _build_app_generate_entity()
app_generate_entity.single_iteration_run = MagicMock()
- pipeline = MagicMock(id="pipe", tenant_id="tenant")
- dataset = SimpleNamespace(id="ds", tenant_id="tenant")
- pipeline.retrieve_dataset.return_value = dataset
- session = MagicMock()
- session.get.return_value = pipeline
- _patch_create_session(mocker, session)
+ _, dataset, _ = _persist_scope(sqlite_session, documents=(_document(),))
+ dataset_ref = module.DatasetRefService.create_dataset_ref(dataset)
+ document_ref = module.DatasetRefService.create_document_ref_from_id(dataset_ref, "doc")
runner = PipelineRunner(
application_generate_entity=app_generate_entity,
queue_manager=MagicMock(),
variable_loader=MagicMock(),
- workflow=MagicMock(),
+ workflow=_workflow(),
system_user_id="sys",
workflow_execution_repository=MagicMock(),
workflow_node_execution_repository=MagicMock(),
)
runner._resolve_user_from = MagicMock(return_value=UserFrom.ACCOUNT)
- runner.get_workflow = MagicMock(
- return_value=MagicMock(
- id="wf",
- tenant_id="tenant",
- app_id="pipe",
- graph_dict={},
- type="rag-pipeline",
- version="v1",
- )
- )
runner._prepare_single_node_execution = MagicMock(return_value=("graph", "pool", "state"))
runner._update_document_status = MagicMock()
runner._handle_event = MagicMock()
- dataset_ref = MagicMock()
- document_ref = MagicMock()
- create_dataset_ref = mocker.patch.object(
- module.DatasetRefService,
- "create_dataset_ref",
- return_value=dataset_ref,
- )
- create_document_ref_from_id = mocker.patch.object(
- module.DatasetRefService,
- "create_document_ref_from_id",
- return_value=document_ref,
- )
-
event = MagicMock()
workflow_entry = MagicMock()
workflow_entry.graph_engine = MagicMock()
@@ -392,34 +368,30 @@ def test_run_single_iteration_path(mocker: MockerFixture):
runner.run()
- create_dataset_ref.assert_called_once_with(dataset)
- create_document_ref_from_id.assert_called_once_with(dataset_ref, "doc")
runner._prepare_single_node_execution.assert_called_once()
runner._update_document_status.assert_called_once_with(event, document_ref)
runner._handle_event.assert_called()
-def test_run_normal_path_builds_graph(mocker: MockerFixture):
+def test_run_normal_path_builds_graph(mocker: MockerFixture, sqlite_session: Session, sqlite_engine: Engine):
app_generate_entity = _build_app_generate_entity()
- pipeline = MagicMock(id="pipe", tenant_id="tenant")
- pipeline.retrieve_dataset.return_value = SimpleNamespace(id="ds", tenant_id="tenant")
- end_user = MagicMock(session_id="sess")
events = []
-
- session = MagicMock()
- session.get.side_effect = [end_user, pipeline]
- _patch_create_session(mocker, session, events=events)
-
- workflow = MagicMock(
- id="wf",
- tenant_id="tenant",
- app_id="pipe",
- graph_dict={"nodes": [], "edges": []},
- environment_variables=[],
- rag_pipeline_variables=[{"variable": "input1", "belong_to_node_id": "start"}],
- type="rag-pipeline",
- version="v1",
+ workflow = _workflow()
+ workflow.rag_pipeline_variables = [
+ {
+ "variable": "input1",
+ "belong_to_node_id": "start",
+ "type": "text-input",
+ "label": "Input",
+ }
+ ]
+ workflow.id = "wf"
+ _persist_scope(
+ sqlite_session,
+ workflow=workflow,
+ end_user=_end_user(),
+ documents=(_document(),),
)
runner = PipelineRunner(
@@ -433,18 +405,10 @@ def test_run_normal_path_builds_graph(mocker: MockerFixture):
)
runner._resolve_user_from = MagicMock(return_value=UserFrom.ACCOUNT)
- runner.get_workflow = MagicMock(return_value=workflow)
runner._init_rag_pipeline_graph = MagicMock(return_value="graph")
runner._update_document_status = MagicMock()
runner._handle_event = MagicMock()
- mocker.patch.object(
- module.RAGPipelineVariable,
- "model_validate",
- return_value=SimpleNamespace(belong_to_node_id="start", variable="input1"),
- )
- mocker.patch.object(module, "RAGPipelineVariableInput", side_effect=lambda **kwargs: SimpleNamespace(**kwargs))
-
class FakeVariablePool:
def add(self, selector, value):
return None
@@ -457,7 +421,15 @@ def test_run_normal_path_builds_graph(mocker: MockerFixture):
mocker.patch.object(module, "WorkflowEntry", return_value=workflow_entry)
mocker.patch.object(module, "WorkflowPersistenceLayer", return_value=MagicMock())
- runner.run()
+ def record_checkin(*_args) -> None:
+ events.append("session_checkin")
+
+ event.listen(sqlite_engine, "checkin", record_checkin)
+ try:
+ runner.run()
+ finally:
+ event.remove(sqlite_engine, "checkin", record_checkin)
- assert events == ["session_enter", "session_exit", "workflow_run"]
+ assert events[-1] == "workflow_run"
+ assert "session_checkin" in events[:-1]
runner._init_rag_pipeline_graph.assert_called_once()