Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

Feature/pytest marker and make targets for dry run (#72)

Louis Choquela3d930b99a3b
diff --git a/Makefile b/Makefile
index fc6c08af4..12cd5a8d4 100644
--- a/Makefile
+++ b/Makefile
@@ -16,6 +16,8 @@ VENV_PIPELEX := $(VIRTUAL_ENV)/bin/pipelex
UV_MIN_VERSION = $(shell grep -m1 'required-version' pyproject.toml | sed -E 's/.*= *"([^<>=, ]+).*/\1/')
+USUAL_PYTEST_MARKERS := "(dry_runable or not (inference or llm or imgg or ocr)) and not (needs_output or pipelex_api)"
+
define PRINT_TITLE
$(eval PROJECT_PART := [$(PROJECT_NAME)])
$(eval TARGET_PART := ($@))
@@ -203,17 +205,17 @@ cleanall: cleanderived cleanenv cleanlibraries
codex-tests: env
$(call PRINT_TITLE,"Unit testing for Codex")
@echo "• Running unit tests for Codex (excluding inference and codex_disabled)"
- $(VENV_PYTEST) --exitfirst --quiet -m "not (inference or codex_disabled or pipelex_api)" || [ $$? = 5 ]
+ $(VENV_PYTEST) -n auto --exitfirst --quiet -m "(dry_runable or not inference) and not (needs_output or pipelex_api)" || [ $$? = 5 ]
gha-tests: env
$(call PRINT_TITLE,"Unit testing for github actions")
@echo "• Running unit tests for github actions (excluding inference and gha_disabled)"
- $(VENV_PYTEST) --exitfirst --quiet -m "not (inference or gha_disabled or pipelex_api)" || [ $$? = 5 ]
+ $(VENV_PYTEST) -n auto --exitfirst --quiet -m "(dry_runable or not inference) and not (gha_disabled or pipelex_api)" || [ $$? = 5 ]
run-all-tests: env
$(call PRINT_TITLE,"Running all unit tests")
@echo "• Running all unit tests"
- $(VENV_PYTEST) --exitfirst --quiet
+ $(VENV_PYTEST) -n auto --exitfirst --quiet
run-manual-trigger-gha-tests: env
$(call PRINT_TITLE,"Running GHA tests")
@@ -229,18 +231,18 @@ test: env
$(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above")
@echo "• Running unit tests"
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) -s -m $(USUAL_PYTEST_MARKERS) -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) -s -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) -s -m $(USUAL_PYTEST_MARKERS) -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
test-xdist: env
$(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above")
@echo "• Running unit tests"
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) -n auto -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) -n auto -m $(USUAL_PYTEST_MARKERS) -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) -n auto -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) -n auto -m $(USUAL_PYTEST_MARKERS) -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
t: test-xdist
@@ -250,9 +252,9 @@ test-quiet: env
$(call PRINT_TITLE,"Unit testing without prints but displaying logs via pytest for WARNING level and above")
@echo "• Running unit tests"
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) -m $(USUAL_PYTEST_MARKERS) -o log_cli=true -o log_level=WARNING -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) -m $(USUAL_PYTEST_MARKERS) -o log_cli=true -o log_level=WARNING $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
tq: test-quiet
@@ -262,9 +264,9 @@ test-with-prints: env
$(call PRINT_TITLE,"Unit testing with prints and our rich logs")
@echo "• Running unit tests"
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) -s -m $(USUAL_PYTEST_MARKERS) -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) -s -m $(USUAL_PYTEST_MARKERS) $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
tp: test-with-prints
@@ -273,9 +275,9 @@ tp: test-with-prints
test-inference: env
$(call PRINT_TITLE,"Unit testing")
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "inference and not imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) --exitfirst -m "inference and not imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "inference and not imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
ti: test-inference
@@ -284,9 +286,9 @@ ti: test-inference
test-ocr: env
$(call PRINT_TITLE,"Unit testing ocr")
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) --exitfirst -m "ocr" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "ocr" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) --exitfirst -m "ocr" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "ocr" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
to: test-ocr
@@ -295,9 +297,9 @@ to: test-ocr
test-imgg: env
$(call PRINT_TITLE,"Unit testing")
@if [ -n "$(TEST)" ]; then \
- $(VENV_PYTEST) --exitfirst -m "imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "imgg" -s -k "$(TEST)" $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
else \
- $(VENV_PYTEST) --exitfirst -m "imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
+ $(VENV_PYTEST) --pipe-run-mode live --exitfirst -m "imgg" -s $(if $(filter 1,$(VERBOSE)),-v,$(if $(filter 2,$(VERBOSE)),-vv,$(if $(filter 3,$(VERBOSE)),-vvv,))); \
fi
tg: test-imgg
diff --git a/pipelex/pipelex.toml b/pipelex/pipelex.toml
index d7a47ff48..79431a752 100644
--- a/pipelex/pipelex.toml
+++ b/pipelex/pipelex.toml
@@ -244,4 +244,4 @@ pipe_stack_limit = 20
[pipelex.dry_run_config]
apply_to_jinja2_rendering = false
-text_gen_truncate_length = 256
\ No newline at end of file
+text_gen_truncate_length = 256
diff --git a/pyproject.toml b/pyproject.toml
index 252d88222..d4f764213 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -208,6 +208,7 @@ markers = [
"ocr: slow and costly due to ocr inference calls",
"gha_disabled: tests that should not run in GitHub Actions",
"codex_disabled: tests that should not run in Codex",
+ "dry_runable: tests that can be run in dry-run mode",
]
minversion = "8.0"
diff --git a/tests/pipelex/pipelex_asynch/test_pipe_batch.py b/tests/pipelex/pipelex_asynch/test_pipe_batch.py
index d2b9eb4a3..5c1d72307 100644
--- a/tests/pipelex/pipelex_asynch/test_pipe_batch.py
+++ b/tests/pipelex/pipelex_asynch/test_pipe_batch.py
@@ -10,6 +10,7 @@ from pipelex.core.working_memory_factory import WorkingMemoryFactory
from pipelex.hub import get_pipe_router, get_pipeline_tracker, get_report_delegate
@pytest.mark.llm
@pytest.mark.inference
@pytest.mark.asyncio(loop_scope="class")
diff --git a/tests/pipelex/pipelex_asynch/test_pipe_imgg.py b/tests/pipelex/pipelex_asynch/test_pipe_imgg.py
index ad501bcfb..56539cff4 100644
--- a/tests/pipelex/pipelex_asynch/test_pipe_imgg.py
+++ b/tests/pipelex/pipelex_asynch/test_pipe_imgg.py
@@ -11,6 +11,7 @@ from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
from tests.pipelex.test_data import IMGGTestCases
@pytest.mark.imgg
@pytest.mark.inference
@pytest.mark.asyncio(loop_scope="class")
diff --git a/tests/pipelex/pipelex_asynch/test_pipe_llm.py b/tests/pipelex/pipelex_asynch/test_pipe_llm.py
index 6b895a69d..1332fd153 100644
--- a/tests/pipelex/pipelex_asynch/test_pipe_llm.py
+++ b/tests/pipelex/pipelex_asynch/test_pipe_llm.py
@@ -16,6 +16,7 @@ from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
from tests.pipelex.test_data import PipeTestCases
@pytest.mark.llm
@pytest.mark.inference
@pytest.mark.asyncio(loop_scope="class")
diff --git a/tests/pipelex/pipelex_asynch/test_pipe_ocr.py b/tests/pipelex/pipelex_asynch/test_pipe_ocr.py
index 9ebe82096..f0915ede5 100644
--- a/tests/pipelex/pipelex_asynch/test_pipe_ocr.py
+++ b/tests/pipelex/pipelex_asynch/test_pipe_ocr.py
@@ -13,6 +13,7 @@ from pipelex.pipe_works.pipe_job_factory import PipeJobFactory
from tests.pipelex.test_data import PipeOcrTestCases
@pytest.mark.ocr
@pytest.mark.inference
@pytest.mark.asyncio(loop_scope="class")
diff --git a/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py b/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py
index ba7ab0c20..e7df2f1a3 100644
--- a/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py
+++ b/tests/pipelex/pipelex_asynch/test_pipe_running_variants.py
@@ -17,6 +17,7 @@ from pipelex.pipeline.job_metadata import JobMetadata
from tests.pipelex.test_data import PipeTestCases
@pytest.mark.llm
@pytest.mark.ocr
@pytest.mark.inference