Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

fix/deep_update-overrides-arrays (#482)

Thomas Hebrard3b0069abd4fd
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c3788d0fe..03d725977 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,11 +2,17 @@
### Unreleased
+**Highlights:** - Previously, in the pipelex config files (`.toml` files in the `.pipelex/` directory, such as `.pipelex/pipelex.toml`, but also the routing profiles files, backends, etc.), when an array was overridden, the new array was concatenated to the old array. Now, the new array overrides the old array.
+
### Changed
- Modified the GHA `version-check.yml` so that the check of the version is only applying to release branches.
- Removed the `pyproject.toml` file from the build.
+### Refactored
+
+- The `find_files_in_dir` function was coded in 3 different places, now it's in `pipelex/tools/misc/file_utils.py`, and accepts `excluded_dirs`.
+
## [v0.16.0] - 2025-11-25
**Highlights:** - Library manager now supports multiple libraries. You can now have multiple libraries in your project, each with its own set of concepts, pipes, and stuffs.
diff --git a/pipelex/libraries/library_utils.py b/pipelex/libraries/library_utils.py
index 1dad303fd..7cae920b9 100644
--- a/pipelex/libraries/library_utils.py
+++ b/pipelex/libraries/library_utils.py
@@ -63,32 +63,6 @@ def get_pipelex_package_dir_for_imports() -> Path | None:
return None
-def find_plx_files_in_dir(dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:
- """Find PLX files matching a pattern in a directory, excluding problematic directories.
-
- Args:
- dir_path: Directory path to search in
- pattern: File pattern to match (e.g. "*.plx")
- is_recursive: Whether to search recursively in subdirectories
-
- Returns:
- List of matching Path objects, filtered to exclude problematic directories
- """
- # Get all files using the base utility
- all_files = find_files_in_dir(dir_path, pattern, is_recursive)
-
- # Filter out files in excluded directories
- filtered_files: list[Path] = []
- excluded_dirs = get_config().pipelex.scan_config.excluded_dirs
- for file_path in all_files:
- # Check if any parent directory is in the exclude list
- should_exclude = any(part in excluded_dirs for part in file_path.parts)
- if not should_exclude:
- filtered_files.append(file_path)
-
- return filtered_files
-
-
def get_pipelex_plx_files_from_dirs(dirs: set[Path]) -> list[Path]:
"""Get all valid Pipelex PLX files from the given directories."""
all_plx_paths: list[Path] = []
@@ -99,10 +73,10 @@ def get_pipelex_plx_files_from_dirs(dirs: set[Path]) -> list[Path]:
continue
# Find all .plx files in the directory, excluding problematic directories
- plx_files = find_plx_files_in_dir(
+ plx_files = find_files_in_dir(
dir_path=str(dir_path),
pattern="*.plx",
- is_recursive=True,
+ excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),
)
# Filter to only include valid Pipelex files
diff --git a/pipelex/system/registries/class_registry_utils.py b/pipelex/system/registries/class_registry_utils.py
index 37c889fa1..9b7b16338 100644
--- a/pipelex/system/registries/class_registry_utils.py
+++ b/pipelex/system/registries/class_registry_utils.py
@@ -1,14 +1,13 @@
import inspect
import sys
import warnings
-from pathlib import Path
from typing import Any
from kajson.kajson_manager import KajsonManager
from pipelex import log
from pipelex.config import get_config
-from pipelex.tools.misc.file_utils import find_files_in_dir as base_find_files_in_dir
+from pipelex.tools.misc.file_utils import find_files_in_dir
from pipelex.tools.typing.module_inspector import (
ModuleFileError,
find_classes_in_module,
@@ -56,10 +55,11 @@ class ClassRegistryUtils:
is_include_imported: Whether to include classes imported from other modules
"""
- python_files = cls.find_files_in_dir(
+ python_files = find_files_in_dir(
dir_path=folder_path,
pattern="*.py",
is_recursive=is_recursive,
+ excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),
)
for python_file in python_files:
@@ -69,33 +69,6 @@ class ClassRegistryUtils:
is_include_imported=is_include_imported,
)
- @classmethod
- def find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:
- """Find files matching a pattern in a directory, excluding common build/cache directories.
-
- Args:
- dir_path: Directory path to search in
- pattern: File pattern to match (e.g. "*.py")
- is_recursive: Whether to search recursively in subdirectories
-
- Returns:
- List of matching Path objects, filtered to exclude problematic directories
-
- """
- # Get all files using the base utility
- all_files = base_find_files_in_dir(dir_path, pattern, is_recursive)
-
- # Filter out files in excluded directories
- filtered_files: list[Path] = []
- excluded_dirs = get_config().pipelex.scan_config.excluded_dirs
- for file_path in all_files:
- # Check if any parent directory is in the exclude list
- should_exclude = any(part in excluded_dirs for part in file_path.parts)
- if not should_exclude:
- filtered_files.append(file_path)
-
- return filtered_files
-
@classmethod
def import_modules_in_folder(
cls,
@@ -120,10 +93,11 @@ class ClassRegistryUtils:
from these base classes. If None, imports all Python files.
"""
- python_files = cls.find_files_in_dir(
+ python_files = find_files_in_dir(
dir_path=folder_path,
pattern="*.py",
is_recursive=is_recursive,
+ excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),
)
for python_file in python_files:
diff --git a/pipelex/system/registries/func_registry_utils.py b/pipelex/system/registries/func_registry_utils.py
index 6f6458131..f71b2e9cd 100644
--- a/pipelex/system/registries/func_registry_utils.py
+++ b/pipelex/system/registries/func_registry_utils.py
@@ -2,13 +2,12 @@ import importlib
import inspect
import pkgutil
from collections.abc import Callable
-from pathlib import Path
from typing import Any
from pipelex import log
from pipelex.config import get_config
from pipelex.system.registries.func_registry import func_registry, pipe_func
-from pipelex.tools.misc.file_utils import find_files_in_dir as base_find_files_in_dir
+from pipelex.tools.misc.file_utils import find_files_in_dir
from pipelex.tools.typing.module_inspector import (
ModuleFileError,
import_module_from_file_if_has_decorated_functions,
@@ -83,10 +82,11 @@ class FuncRegistryUtils:
is_recursive: Whether to search recursively in subdirectories
"""
- python_files = cls._find_files_in_dir(
+ python_files = find_files_in_dir(
dir_path=folder_path,
pattern="*.py",
is_recursive=is_recursive,
+ excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),
)
for python_file in python_files:
@@ -188,30 +188,3 @@ class FuncRegistryUtils:
"""
custom_name = getattr(func, "_pipe_func_name", None)
return custom_name if custom_name is not None else func.__name__
-
- @classmethod
- def _find_files_in_dir(cls, dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:
- """Find files matching a pattern in a directory, excluding common build/cache directories.
-
- Args:
- dir_path: Directory path to search in
- pattern: File pattern to match (e.g. "*.py")
- is_recursive: Whether to search recursively in subdirectories
-
- Returns:
- List of matching Path objects, filtered to exclude problematic directories
-
- """
- # Get all files using the base utility
- all_files = base_find_files_in_dir(dir_path, pattern, is_recursive)
-
- # Filter out files in excluded directories
- filtered_files: list[Path] = []
- excluded_dirs = get_config().pipelex.scan_config.excluded_dirs
- for file_path in all_files:
- # Check if any parent directory is in the exclude list
- should_exclude = any(part in excluded_dirs for part in file_path.parts)
- if not should_exclude:
- filtered_files.append(file_path)
-
- return filtered_files
diff --git a/pipelex/tools/misc/file_utils.py b/pipelex/tools/misc/file_utils.py
index 21f30aae4..28c0c1f78 100644
--- a/pipelex/tools/misc/file_utils.py
+++ b/pipelex/tools/misc/file_utils.py
@@ -347,19 +347,26 @@ def get_incremental_file_path(
########################################################
-def find_files_in_dir(dir_path: str, pattern: str, is_recursive: bool) -> list[Path]:
+def find_files_in_dir(dir_path: str, pattern: str, is_recursive: bool = True, excluded_dirs: list[str] | None = None) -> list[Path]:
"""Find files matching a pattern in a directory.
Args:
dir_path: Directory path to search in
pattern: File pattern to match (e.g. "*.py")
is_recursive: Whether to search recursively in subdirectories
-
+ excluded_dirs: List of directories to exclude from the search
Returns:
List of matching Path objects
"""
path = Path(dir_path)
+ files: list[Path] = []
if is_recursive:
- return list(path.rglob(pattern))
- return list(path.glob(pattern))
+ files = list(path.rglob(pattern))
+ else:
+ files = list(path.glob(pattern))
+
+ for file in files:
+ if excluded_dirs is not None and any(part in excluded_dirs for part in file.parts):
+ files.remove(file)
+ return files
diff --git a/pipelex/tools/misc/json_utils.py b/pipelex/tools/misc/json_utils.py
index 40842e3fd..a2d59eab6 100644
--- a/pipelex/tools/misc/json_utils.py
+++ b/pipelex/tools/misc/json_utils.py
@@ -157,8 +157,8 @@ def deep_update(target_dict: dict[str, Any], updates: dict[str, Any]):
"""Recursively updates a dictionary with values from another dictionary.
This function performs a deep merge of two dictionaries, handling nested
- dictionaries and lists. For dictionaries, it recursively updates values,
- and for lists, it concatenates them.
+ dictionaries. For dictionaries, it recursively updates values. For all other
+ types (including lists), values from updates override the target values.
Args:
target_dict (Dict[str, Any]): The dictionary to update. This dictionary
@@ -170,14 +170,12 @@ def deep_update(target_dict: dict[str, Any], updates: dict[str, Any]):
>>> updates = {"b": {"y": 4, "z": 5}, "c": [3, 4]}
>>> deep_update(base, updates)
>>> print(base)
- {'a': 1, 'b': {'x': 2, 'y': 4, 'z': 5}, 'c': [1, 2, 3, 4]}
+ {'a': 1, 'b': {'x': 2, 'y': 4, 'z': 5}, 'c': [3, 4]}
"""
for key, value in updates.items():
if isinstance(value, dict) and key in target_dict and isinstance(target_dict[key], dict):
deep_update(target_dict[key], value) # pyright: ignore[reportUnknownArgumentType]
- elif isinstance(value, list) and key in target_dict and isinstance(target_dict[key], list):
- target_dict[key] = list(target_dict[key] + value)
else:
target_dict[key] = value
diff --git a/tests/integration/pipelex/tools/misc/test_json_utils.py b/tests/integration/pipelex/tools/misc/test_json_utils.py
new file mode 100644
index 000000000..76d7005d5
--- /dev/null
+++ b/tests/integration/pipelex/tools/misc/test_json_utils.py
@@ -0,0 +1,250 @@
+from typing import Any
+
+from pipelex.tools.misc.json_utils import deep_update
+
+
+class TestDeepUpdate:
+ def test_simple_key_update(self):
+ """Test updating simple key-value pairs."""
+ target = {"key_a": 1, "key_b": 2}
+ updates = {"key_b": 3}
+ deep_update(target, updates)
+ assert target == {"key_a": 1, "key_b": 3}
+
+ def test_add_new_key(self):
+ """Test adding new keys to target dictionary."""
+ target = {"key_a": 1}
+ updates = {"key_b": 2, "key_c": 3}
+ deep_update(target, updates)
+ assert target == {"key_a": 1, "key_b": 2, "key_c": 3}
+
+ def test_nested_dict_update(self):
+ """Test deep update of nested dictionaries."""
+ target = {"key_a": 1, "key_b": {"key_x": 2, "key_y": 3}}
+ updates = {"key_b": {"key_y": 4, "key_z": 5}}
+ deep_update(target, updates)
+ assert target == {"key_a": 1, "key_b": {"key_x": 2, "key_y": 4, "key_z": 5}}
+
+ def test_list_override(self):
+ """Test that lists are overridden, not concatenated."""
+ target = {"key_a": [1, 2, 3]}
+ updates = {"key_a": [4, 5]}
+ deep_update(target, updates)
+ assert target == {"key_a": [4, 5]}
+
+ def test_list_override_in_nested_dict(self):
+ """Test that lists in nested dictionaries are overridden."""
+ target = {"key_a": {"key_b": [1, 2, 3]}}
+ updates = {"key_a": {"key_b": [4, 5]}}
+ deep_update(target, updates)
+ assert target == {"key_a": {"key_b": [4, 5]}}
+
+ def test_deeply_nested_dict_update(self):
+ """Test update of deeply nested dictionaries."""
+ target = {
+ "level_1": {
+ "level_2": {
+ "level_3": {
+ "key_a": 1,
+ "key_b": 2,
+ }
+ }
+ }
+ }
+ updates = {
+ "level_1": {
+ "level_2": {
+ "level_3": {
+ "key_b": 3,
+ "key_c": 4,
+ }
+ }
+ }
+ }
+ deep_update(target, updates)
+ expected = {
+ "level_1": {
+ "level_2": {
+ "level_3": {
+ "key_a": 1,
+ "key_b": 3,
+ "key_c": 4,
+ }
+ }
+ }
+ }
+ assert target == expected
+
+ def test_mixed_types_update(self):
+ """Test updating with mixed types (primitives, dicts, lists)."""
+ target = {
+ "string": "old",
+ "number": 1,
+ "nested": {"key_a": 1},
+ "array": [1, 2],
+ }
+ updates = {
+ "string": "new",
+ "number": 2,
+ "nested": {"key_b": 2},
+ "array": [3, 4, 5],
+ }
+ deep_update(target, updates)
+ expected = {
+ "string": "new",
+ "number": 2,
+ "nested": {"key_a": 1, "key_b": 2},
+ "array": [3, 4, 5],
+ }
+ assert target == expected
+
+ def test_empty_updates(self):
+ """Test that empty updates dictionary doesn't change target."""
+ target = {"key_a": 1, "key_b": 2}
+ updates: dict[str, Any] = {}
+ deep_update(target, updates)
+ assert target == {"key_a": 1, "key_b": 2}
+
+ def test_empty_target(self):
+ """Test updating an empty target dictionary."""
+ target: dict[str, Any] = {}
+ updates = {"key_a": 1, "key_b": {"key_c": 2}}
+ deep_update(target, updates)
+ assert target == {"key_a": 1, "key_b": {"key_c": 2}}
+
+ def test_replace_dict_with_primitive(self):
+ """Test replacing a dictionary value with a primitive."""
+ target = {"key_a": {"key_b": 1}}
+ updates = {"key_a": "string"}
+ deep_update(target, updates)
+ assert target == {"key_a": "string"}
+
+ def test_replace_primitive_with_dict(self):
+ """Test replacing a primitive value with a dictionary."""
+ target = {"key_a": "string"}
+ updates = {"key_a": {"key_b": 1}}
+ deep_update(target, updates)
+ assert target == {"key_a": {"key_b": 1}}
+
+ def test_replace_list_with_primitive(self):
+ """Test replacing a list with a primitive value."""
+ target = {"key_a": [1, 2, 3]}
+ updates = {"key_a": "string"}
+ deep_update(target, updates)
+ assert target == {"key_a": "string"}
+
+ def test_replace_primitive_with_list(self):
+ """Test replacing a primitive value with a list."""
+ target = {"key_a": "string"}
+ updates = {"key_a": [1, 2, 3]}
+ deep_update(target, updates)
+ assert target == {"key_a": [1, 2, 3]}
+
+ def test_none_values(self):
+ """Test handling of None values."""
+ target = {"key_a": 1, "key_b": None}
+ updates = {"key_a": None, "key_b": 2}
+ deep_update(target, updates)
+ assert target == {"key_a": None, "key_b": 2}
+
+ def test_empty_list_override(self):
+ """Test that empty lists override non-empty lists."""
+ target = {"key_a": [1, 2, 3]}
+ updates: dict[str, Any] = {"key_a": []}
+ deep_update(target, updates)
+ assert target == {"key_a": []}
+
+ def test_empty_dict_merge(self):
+ """Test merging with empty nested dictionaries."""
+ target = {"key_a": {"key_b": 1}}
+ updates: dict[str, Any] = {"key_a": {}}
+ deep_update(target, updates)
+ # Empty dict should merge but not change existing keys
+ assert target == {"key_a": {"key_b": 1}}
+
+ def test_complex_nested_structure(self):
+ """Test a complex nested structure with multiple levels and types."""
+ target = {
+ "config": {
+ "database": {
+ "host": "localhost",
+ "port": 5432,
+ "credentials": {
+ "username": "admin",
+ },
+ },
+ "features": ["feature_a", "feature_b"],
+ },
+ "version": "1.0.0",
+ }
+ updates = {
+ "config": {
+ "database": {
+ "port": 3306,
+ "credentials": {
+ "password": "secret",
+ },
+ },
+ "features": ["feature_c"],
+ "cache": {"enabled": True},
+ },
+ "version": "1.1.0",
+ }
+ deep_update(target, updates)
+ expected = {
+ "config": {
+ "database": {
+ "host": "localhost",
+ "port": 3306,
+ "credentials": {
+ "username": "admin",
+ "password": "secret",
+ },
+ },
+ "features": ["feature_c"],
+ "cache": {"enabled": True},
+ },
+ "version": "1.1.0",
+ }
+ assert target == expected
+
+ def test_boolean_values(self):
+ """Test handling of boolean values."""
+ target = {"flag_a": True, "flag_b": False}
+ updates = {"flag_a": False, "flag_b": True}
+ deep_update(target, updates)
+ assert target == {"flag_a": False, "flag_b": True}
+
+ def test_numeric_types(self):
+ """Test handling of different numeric types."""
+ target = {"int_val": 1, "float_val": 1.5}
+ updates = {"int_val": 2.5, "float_val": 3}
+ deep_update(target, updates)
+ assert target == {"int_val": 2.5, "float_val": 3}
+
+ def test_list_of_dicts(self):
+ """Test that lists of dictionaries are overridden, not merged."""
+ target = {"items": [{"id": 1, "name": "item1"}, {"id": 2, "name": "item2"}]}
+ updates = {"items": [{"id": 3, "name": "item3"}]}
+ deep_update(target, updates)
+ assert target == {"items": [{"id": 3, "name": "item3"}]}
+
+ def test_multiple_level_additions(self):
+ """Test adding keys at multiple levels simultaneously."""
+ target = {"level_1": {"level_2": {"existing": "value"}}}
+ updates = {
+ "level_1": {
+ "level_2": {"new_key": "new_value"},
+ "new_level": {"key": "value"},
+ },
+ "new_top": "value",
+ }
+ deep_update(target, updates)
+ expected = {
+ "level_1": {
+ "level_2": {"existing": "value", "new_key": "new_value"},
+ "new_level": {"key": "value"},
+ },
+ "new_top": "value",
+ }
+ assert target == expected
diff --git a/tests/unit/pipelex/tools/test_class_registry_utils.py b/tests/unit/pipelex/tools/test_class_registry_utils.py
index 846639f60..c536b6bd3 100644
--- a/tests/unit/pipelex/tools/test_class_registry_utils.py
+++ b/tests/unit/pipelex/tools/test_class_registry_utils.py
@@ -43,9 +43,17 @@ class TestClassRegistryUtilsUnit:
def test_register_classes_in_folder_unit(self, mocker: MockerFixture):
"""Unit test for registering classes from a folder using mocks."""
+ # Mock get_config to return excluded_dirs
+ mock_config = mocker.MagicMock()
+ mock_config.pipelex.scan_config.excluded_dirs = ["__pycache__", ".git"]
+ mocker.patch("pipelex.system.registries.class_registry_utils.get_config", return_value=mock_config)
+
# Mock the file finding and registration
mock_files = [Path("/fake/file1.py"), Path("/fake/file2.py")]
- mock_find_files = mocker.patch.object(ClassRegistryUtils, "find_files_in_dir", return_value=mock_files)
+ mock_find_files = mocker.patch(
+ "pipelex.system.registries.class_registry_utils.find_files_in_dir",
+ return_value=mock_files,
+ )
mock_register_file = mocker.patch.object(ClassRegistryUtils, "register_classes_in_file")
ClassRegistryUtils.register_classes_in_folder(
@@ -56,7 +64,12 @@ class TestClassRegistryUtilsUnit:
)
# Verify find_files_in_dir was called correctly
- mock_find_files.assert_called_once_with(dir_path="/fake/folder", pattern="*.py", is_recursive=True)
+ mock_find_files.assert_called_once_with(
+ dir_path="/fake/folder",
+ pattern="*.py",
+ is_recursive=True,
+ excluded_dirs=["__pycache__", ".git"],
+ )
# Verify register_classes_in_file was called for each file
assert mock_register_file.call_count == 2