Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
fix/deep_update-overrides-arrays (#482)
Thomas Hebrard3b0069abd4fd
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex c3788d0fe..03d725977 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -2,11 +2,17 @@### Unreleased+**Highlights:** - Previously, in the pipelex config files (`.toml` files in the `.pipelex/` directory, such as `.pipelex/pipelex.toml`, but also the routing profiles files, backends, etc.), when an array was overridden, the new array was concatenated to the old array. Now, the new array overrides the old array.+### Changed- Modified the GHA `version-check.yml` so that the check of the version is only applying to release branches.- Removed the `pyproject.toml` file from the build.+### Refactored++- The `find_files_in_dir` function was coded in 3 different places, now it's in `pipelex/tools/misc/file_utils.py`, and accepts `excluded_dirs`.+## [v0.16.0] - 2025-11-25**Highlights:** - Library manager now supports multiple libraries. You can now have multiple libraries in your project, each with its own set of concepts, pipes, and stuffs.diff --git a/pipelex/libraries/library_utils.py b/pipelex/libraries/library_utils.pyindex 1dad303fd..7cae920b9 100644--- a/pipelex/libraries/library_utils.py+++ b/pipelex/libraries/library_utils.py@@ -63,32 +63,6 @@ def get_pipelex_package_dir_for_imports() -> Path | None:return None-def find_plx_files_in_dir(dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:- """Find PLX files matching a pattern in a directory, excluding problematic directories.-- Args:- dir_path: Directory path to search in- pattern: File pattern to match (e.g. "*.plx")- is_recursive: Whether to search recursively in subdirectories-- Returns:- List of matching Path objects, filtered to exclude problematic directories- """- # Get all files using the base utility- all_files = find_files_in_dir(dir_path, pattern, is_recursive)-- # Filter out files in excluded directories- filtered_files: list[Path] = []- excluded_dirs = get_config().pipelex.scan_config.excluded_dirs- for file_path in all_files:- # Check if any parent directory is in the exclude list- should_exclude = any(part in excluded_dirs for part in file_path.parts)- if not should_exclude:- filtered_files.append(file_path)-- return filtered_files--def get_pipelex_plx_files_from_dirs(dirs: set[Path]) -> list[Path]:"""Get all valid Pipelex PLX files from the given directories."""all_plx_paths: list[Path] = []@@ -99,10 +73,10 @@ def get_pipelex_plx_files_from_dirs(dirs: set[Path]) -> list[Path]:continue# Find all .plx files in the directory, excluding problematic directories- plx_files = find_plx_files_in_dir(+ plx_files = find_files_in_dir(dir_path=str(dir_path),pattern="*.plx",- is_recursive=True,+ excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),)# Filter to only include valid Pipelex filesdiff --git a/pipelex/system/registries/class_registry_utils.py b/pipelex/system/registries/class_registry_utils.pyindex 37c889fa1..9b7b16338 100644--- a/pipelex/system/registries/class_registry_utils.py+++ b/pipelex/system/registries/class_registry_utils.py@@ -1,14 +1,13 @@import inspectimport sysimport warnings-from pathlib import Pathfrom typing import Anyfrom kajson.kajson_manager import KajsonManagerfrom pipelex import logfrom pipelex.config import get_config-from pipelex.tools.misc.file_utils import find_files_in_dir as base_find_files_in_dir+from pipelex.tools.misc.file_utils import find_files_in_dirfrom pipelex.tools.typing.module_inspector import (ModuleFileError,find_classes_in_module,@@ -56,10 +55,11 @@ class ClassRegistryUtils:is_include_imported: Whether to include classes imported from other modules"""- python_files = cls.find_files_in_dir(+ python_files = find_files_in_dir(dir_path=folder_path,pattern="*.py",is_recursive=is_recursive,+ excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),)for python_file in python_files:@@ -69,33 +69,6 @@ class ClassRegistryUtils:is_include_imported=is_include_imported,)- @classmethod- def find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:- """Find files matching a pattern in a directory, excluding common build/cache directories.-- Args:- dir_path: Directory path to search in- pattern: File pattern to match (e.g. "*.py")- is_recursive: Whether to search recursively in subdirectories-- Returns:- List of matching Path objects, filtered to exclude problematic directories-- """- # Get all files using the base utility- all_files = base_find_files_in_dir(dir_path, pattern, is_recursive)-- # Filter out files in excluded directories- filtered_files: list[Path] = []- excluded_dirs = get_config().pipelex.scan_config.excluded_dirs- for file_path in all_files:- # Check if any parent directory is in the exclude list- should_exclude = any(part in excluded_dirs for part in file_path.parts)- if not should_exclude:- filtered_files.append(file_path)-- return filtered_files-@classmethoddef import_modules_in_folder(cls,@@ -120,10 +93,11 @@ class ClassRegistryUtils:from these base classes. If None, imports all Python files."""- python_files = cls.find_files_in_dir(+ python_files = find_files_in_dir(dir_path=folder_path,pattern="*.py",is_recursive=is_recursive,+ excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),)for python_file in python_files:diff --git a/pipelex/system/registries/func_registry_utils.py b/pipelex/system/registries/func_registry_utils.pyindex 6f6458131..f71b2e9cd 100644--- a/pipelex/system/registries/func_registry_utils.py+++ b/pipelex/system/registries/func_registry_utils.py@@ -2,13 +2,12 @@ import importlibimport inspectimport pkgutilfrom collections.abc import Callable-from pathlib import Pathfrom typing import Anyfrom pipelex import logfrom pipelex.config import get_configfrom pipelex.system.registries.func_registry import func_registry, pipe_func-from pipelex.tools.misc.file_utils import find_files_in_dir as base_find_files_in_dir+from pipelex.tools.misc.file_utils import find_files_in_dirfrom pipelex.tools.typing.module_inspector import (ModuleFileError,import_module_from_file_if_has_decorated_functions,@@ -83,10 +82,11 @@ class FuncRegistryUtils:is_recursive: Whether to search recursively in subdirectories"""- python_files = cls._find_files_in_dir(+ python_files = find_files_in_dir(dir_path=folder_path,pattern="*.py",is_recursive=is_recursive,+ excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),)for python_file in python_files:@@ -188,30 +188,3 @@ class FuncRegistryUtils:"""custom_name = getattr(func, "_pipe_func_name", None)return custom_name if custom_name is not None else func.__name__-- @classmethod- def _find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:- """Find files matching a pattern in a directory, excluding common build/cache directories.-- Args:- dir_path: Directory path to search in- pattern: File pattern to match (e.g. "*.py")- is_recursive: Whether to search recursively in subdirectories-- Returns:- List of matching Path objects, filtered to exclude problematic directories-- """- # Get all files using the base utility- all_files = base_find_files_in_dir(dir_path, pattern, is_recursive)-- # Filter out files in excluded directories- filtered_files: list[Path] = []- excluded_dirs = get_config().pipelex.scan_config.excluded_dirs- for file_path in all_files:- # Check if any parent directory is in the exclude list- should_exclude = any(part in excluded_dirs for part in file_path.parts)- if not should_exclude:- filtered_files.append(file_path)-- return filtered_filesdiff --git a/pipelex/tools/misc/file_utils.py b/pipelex/tools/misc/file_utils.pyindex 21f30aae4..28c0c1f78 100644--- a/pipelex/tools/misc/file_utils.py+++ b/pipelex/tools/misc/file_utils.py@@ -347,19 +347,26 @@ def get_incremental_file_path(########################################################-def find_files_in_dir(dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:+def find_files_in_dir(dir_path: str, pattern: str, is_recursive: bool = True, excluded_dirs: list[str] | None = None) -> list[Path]:"""Find files matching a pattern in a directory.Args:dir_path: Directory path to search inpattern: File pattern to match (e.g. "*.py")is_recursive: Whether to search recursively in subdirectories-+ excluded_dirs: List of directories to exclude from the searchReturns:List of matching Path objects"""path = Path(dir_path)+ files: list[Path] = []if is_recursive:- return list(path.rglob(pattern))- return list(path.glob(pattern))+ files = list(path.rglob(pattern))+ else:+ files = list(path.glob(pattern))++ for file in files:+ if excluded_dirs is not None and any(part in excluded_dirs for part in file.parts):+ files.remove(file)+ return filesdiff --git a/pipelex/tools/misc/json_utils.py b/pipelex/tools/misc/json_utils.pyindex 40842e3fd..a2d59eab6 100644--- a/pipelex/tools/misc/json_utils.py+++ b/pipelex/tools/misc/json_utils.py@@ -157,8 +157,8 @@ def deep_update(target_dict: dict[str, Any], updates: dict[str, Any]):"""Recursively updates a dictionary with values from another dictionary.This function performs a deep merge of two dictionaries, handling nested- dictionaries and lists. For dictionaries, it recursively updates values,- and for lists, it concatenates them.+ dictionaries. For dictionaries, it recursively updates values. For all other+ types (including lists), values from updates override the target values.Args:target_dict (Dict[str, Any]): The dictionary to update. This dictionary@@ -170,14 +170,12 @@ def deep_update(target_dict: dict[str, Any], updates: dict[str, Any]):>>> updates = {"b": {"y": 4, "z": 5}, "c": [3, 4]}>>> deep_update(base, updates)>>> print(base)- {'a': 1, 'b': {'x': 2, 'y': 4, 'z': 5}, 'c': [1, 2, 3, 4]}+ {'a': 1, 'b': {'x': 2, 'y': 4, 'z': 5}, 'c': [3, 4]}"""for key, value in updates.items():if isinstance(value, dict) and key in target_dict and isinstance(target_dict[key], dict):deep_update(target_dict[key], value) # pyright: ignore[reportUnknownArgumentType]- elif isinstance(value, list) and key in target_dict and isinstance(target_dict[key], list):- target_dict[key] = list(target_dict[key] + value)else:target_dict[key] = valuediff --git a/tests/integration/pipelex/tools/misc/test_json_utils.py b/tests/integration/pipelex/tools/misc/test_json_utils.pynew file mode 100644index 000000000..76d7005d5--- /dev/null+++ b/tests/integration/pipelex/tools/misc/test_json_utils.py@@ -0,0 +1,250 @@+from typing import Any++from pipelex.tools.misc.json_utils import deep_update+++class TestDeepUpdate:+ def test_simple_key_update(self):+ """Test updating simple key-value pairs."""+ target = {"key_a": 1, "key_b": 2}+ updates = {"key_b": 3}+ deep_update(target, updates)+ assert target == {"key_a": 1, "key_b": 3}++ def test_add_new_key(self):+ """Test adding new keys to target dictionary."""+ target = {"key_a": 1}+ updates = {"key_b": 2, "key_c": 3}+ deep_update(target, updates)+ assert target == {"key_a": 1, "key_b": 2, "key_c": 3}++ def test_nested_dict_update(self):+ """Test deep update of nested dictionaries."""+ target = {"key_a": 1, "key_b": {"key_x": 2, "key_y": 3}}+ updates = {"key_b": {"key_y": 4, "key_z": 5}}+ deep_update(target, updates)+ assert target == {"key_a": 1, "key_b": {"key_x": 2, "key_y": 4, "key_z": 5}}++ def test_list_override(self):+ """Test that lists are overridden, not concatenated."""+ target = {"key_a": [1, 2, 3]}+ updates = {"key_a": [4, 5]}+ deep_update(target, updates)+ assert target == {"key_a": [4, 5]}++ def test_list_override_in_nested_dict(self):+ """Test that lists in nested dictionaries are overridden."""+ target = {"key_a": {"key_b": [1, 2, 3]}}+ updates = {"key_a": {"key_b": [4, 5]}}+ deep_update(target, updates)+ assert target == {"key_a": {"key_b": [4, 5]}}++ def test_deeply_nested_dict_update(self):+ """Test update of deeply nested dictionaries."""+ target = {+ "level_1": {+ "level_2": {+ "level_3": {+ "key_a": 1,+ "key_b": 2,+ }+ }+ }+ }+ updates = {+ "level_1": {+ "level_2": {+ "level_3": {+ "key_b": 3,+ "key_c": 4,+ }+ }+ }+ }+ deep_update(target, updates)+ expected = {+ "level_1": {+ "level_2": {+ "level_3": {+ "key_a": 1,+ "key_b": 3,+ "key_c": 4,+ }+ }+ }+ }+ assert target == expected++ def test_mixed_types_update(self):+ """Test updating with mixed types (primitives, dicts, lists)."""+ target = {+ "string": "old",+ "number": 1,+ "nested": {"key_a": 1},+ "array": [1, 2],+ }+ updates = {+ "string": "new",+ "number": 2,+ "nested": {"key_b": 2},+ "array": [3, 4, 5],+ }+ deep_update(target, updates)+ expected = {+ "string": "new",+ "number": 2,+ "nested": {"key_a": 1, "key_b": 2},+ "array": [3, 4, 5],+ }+ assert target == expected++ def test_empty_updates(self):+ """Test that empty updates dictionary doesn't change target."""+ target = {"key_a": 1, "key_b": 2}+ updates: dict[str, Any] = {}+ deep_update(target, updates)+ assert target == {"key_a": 1, "key_b": 2}++ def test_empty_target(self):+ """Test updating an empty target dictionary."""+ target: dict[str, Any] = {}+ updates = {"key_a": 1, "key_b": {"key_c": 2}}+ deep_update(target, updates)+ assert target == {"key_a": 1, "key_b": {"key_c": 2}}++ def test_replace_dict_with_primitive(self):+ """Test replacing a dictionary value with a primitive."""+ target = {"key_a": {"key_b": 1}}+ updates = {"key_a": "string"}+ deep_update(target, updates)+ assert target == {"key_a": "string"}++ def test_replace_primitive_with_dict(self):+ """Test replacing a primitive value with a dictionary."""+ target = {"key_a": "string"}+ updates = {"key_a": {"key_b": 1}}+ deep_update(target, updates)+ assert target == {"key_a": {"key_b": 1}}++ def test_replace_list_with_primitive(self):+ """Test replacing a list with a primitive value."""+ target = {"key_a": [1, 2, 3]}+ updates = {"key_a": "string"}+ deep_update(target, updates)+ assert target == {"key_a": "string"}++ def test_replace_primitive_with_list(self):+ """Test replacing a primitive value with a list."""+ target = {"key_a": "string"}+ updates = {"key_a": [1, 2, 3]}+ deep_update(target, updates)+ assert target == {"key_a": [1, 2, 3]}++ def test_none_values(self):+ """Test handling of None values."""+ target = {"key_a": 1, "key_b": None}+ updates = {"key_a": None, "key_b": 2}+ deep_update(target, updates)+ assert target == {"key_a": None, "key_b": 2}++ def test_empty_list_override(self):+ """Test that empty lists override non-empty lists."""+ target = {"key_a": [1, 2, 3]}+ updates: dict[str, Any] = {"key_a": []}+ deep_update(target, updates)+ assert target == {"key_a": []}++ def test_empty_dict_merge(self):+ """Test merging with empty nested dictionaries."""+ target = {"key_a": {"key_b": 1}}+ updates: dict[str, Any] = {"key_a": {}}+ deep_update(target, updates)+ # Empty dict should merge but not change existing keys+ assert target == {"key_a": {"key_b": 1}}++ def test_complex_nested_structure(self):+ """Test a complex nested structure with multiple levels and types."""+ target = {+ "config": {+ "database": {+ "host": "localhost",+ "port": 5432,+ "credentials": {+ "username": "admin",+ },+ },+ "features": ["feature_a", "feature_b"],+ },+ "version": "1.0.0",+ }+ updates = {+ "config": {+ "database": {+ "port": 3306,+ "credentials": {+ "password": "secret",+ },+ },+ "features": ["feature_c"],+ "cache": {"enabled": True},+ },+ "version": "1.1.0",+ }+ deep_update(target, updates)+ expected = {+ "config": {+ "database": {+ "host": "localhost",+ "port": 3306,+ "credentials": {+ "username": "admin",+ "password": "secret",+ },+ },+ "features": ["feature_c"],+ "cache": {"enabled": True},+ },+ "version": "1.1.0",+ }+ assert target == expected++ def test_boolean_values(self):+ """Test handling of boolean values."""+ target = {"flag_a": True, "flag_b": False}+ updates = {"flag_a": False, "flag_b": True}+ deep_update(target, updates)+ assert target == {"flag_a": False, "flag_b": True}++ def test_numeric_types(self):+ """Test handling of different numeric types."""+ target = {"int_val": 1, "float_val": 1.5}+ updates = {"int_val": 2.5, "float_val": 3}+ deep_update(target, updates)+ assert target == {"int_val": 2.5, "float_val": 3}++ def test_list_of_dicts(self):+ """Test that lists of dictionaries are overridden, not merged."""+ target = {"items": [{"id": 1, "name": "item1"}, {"id": 2, "name": "item2"}]}+ updates = {"items": [{"id": 3, "name": "item3"}]}+ deep_update(target, updates)+ assert target == {"items": [{"id": 3, "name": "item3"}]}++ def test_multiple_level_additions(self):+ """Test adding keys at multiple levels simultaneously."""+ target = {"level_1": {"level_2": {"existing": "value"}}}+ updates = {+ "level_1": {+ "level_2": {"new_key": "new_value"},+ "new_level": {"key": "value"},+ },+ "new_top": "value",+ }+ deep_update(target, updates)+ expected = {+ "level_1": {+ "level_2": {"existing": "value", "new_key": "new_value"},+ "new_level": {"key": "value"},+ },+ "new_top": "value",+ }+ assert target == expecteddiff --git a/tests/unit/pipelex/tools/test_class_registry_utils.py b/tests/unit/pipelex/tools/test_class_registry_utils.pyindex 846639f60..c536b6bd3 100644--- a/tests/unit/pipelex/tools/test_class_registry_utils.py+++ b/tests/unit/pipelex/tools/test_class_registry_utils.py@@ -43,9 +43,17 @@ class TestClassRegistryUtilsUnit:def test_register_classes_in_folder_unit(self, mocker: MockerFixture):"""Unit test for registering classes from a folder using mocks."""+ # Mock get_config to return excluded_dirs+ mock_config = mocker.MagicMock()+ mock_config.pipelex.scan_config.excluded_dirs = ["__pycache__", ".git"]+ mocker.patch("pipelex.system.registries.class_registry_utils.get_config", return_value=mock_config)+# Mock the file finding and registrationmock_files = [Path("/fake/file1.py"), Path("/fake/file2.py")]- mock_find_files = mocker.patch.object(ClassRegistryUtils, "find_files_in_dir", return_value=mock_files)+ mock_find_files = mocker.patch(+ "pipelex.system.registries.class_registry_utils.find_files_in_dir",+ return_value=mock_files,+ )mock_register_file = mocker.patch.object(ClassRegistryUtils, "register_classes_in_file")ClassRegistryUtils.register_classes_in_folder(@@ -56,7 +64,12 @@ class TestClassRegistryUtilsUnit:)# Verify find_files_in_dir was called correctly- mock_find_files.assert_called_once_with(dir_path="/fake/folder", pattern="*.py", is_recursive=True)+ mock_find_files.assert_called_once_with(+ dir_path="/fake/folder",+ pattern="*.py",+ is_recursive=True,+ excluded_dirs=["__pycache__", ".git"],+ )# Verify register_classes_in_file was called for each fileassert mock_register_file.call_count == 2