Skip to content

Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.

fix/excluded-dirs-pipe-builder (#495)

Thomas Hebrard85d5fa49b8ca
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4ebd06c3a..859ed1730 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,11 @@
# Changelog
+### Unreleased
+
+### Added
+
+- Added `force_include_dirs` parameter to the `find_files_in_dir` function. This allows specific directories to be force included in the search even when they are nested within excluded directories. For example, you can now exclude `.venv` while still including `.venv/lib/python3.11/site-packages/pipelex` for loading Pipelex libraries from installed packages.
+
## [v0.17.1] - 2025-11-27
### Fixed
diff --git a/pipelex/libraries/library_utils.py b/pipelex/libraries/library_utils.py
index 7cae920b9..5d8d1ec4a 100644
--- a/pipelex/libraries/library_utils.py
+++ b/pipelex/libraries/library_utils.py
@@ -77,6 +77,7 @@ def get_pipelex_plx_files_from_dirs(dirs: set[Path]) -> list[Path]:
dir_path=str(dir_path),
pattern="*.plx",
excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),
+ force_include_dirs=[str(dir_path) for dir_path in dirs],
)
# Filter to only include valid Pipelex files
diff --git a/pipelex/system/registries/class_registry_utils.py b/pipelex/system/registries/class_registry_utils.py
index 2b0dc2eb4..474615597 100644
--- a/pipelex/system/registries/class_registry_utils.py
+++ b/pipelex/system/registries/class_registry_utils.py
@@ -97,6 +97,7 @@ class ClassRegistryUtils:
pattern="*.py",
is_recursive=is_recursive,
excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),
+ force_include_dirs=[folder_path],
)
for python_file in python_files:
diff --git a/pipelex/system/registries/func_registry_utils.py b/pipelex/system/registries/func_registry_utils.py
index f71b2e9cd..668920e91 100644
--- a/pipelex/system/registries/func_registry_utils.py
+++ b/pipelex/system/registries/func_registry_utils.py
@@ -87,6 +87,7 @@ class FuncRegistryUtils:
pattern="*.py",
is_recursive=is_recursive,
excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),
+ force_include_dirs=[folder_path],
)
for python_file in python_files:
diff --git a/pipelex/tools/misc/file_utils.py b/pipelex/tools/misc/file_utils.py
index 281b7c839..7e67ffb11 100644
--- a/pipelex/tools/misc/file_utils.py
+++ b/pipelex/tools/misc/file_utils.py
@@ -345,14 +345,23 @@ def get_incremental_file_path(
########################################################
-def find_files_in_dir(dir_path: str, pattern: str, is_recursive: bool = True, excluded_dirs: list[str] | None = None) -> list[Path]:
+def find_files_in_dir(
+ dir_path: str,
+ pattern: str,
+ is_recursive: bool = True,
+ excluded_dirs: list[str] | None = None,
+ force_include_dirs: list[str] | None = None,
+) -> list[Path]:
"""Find files matching a pattern in a directory.
Args:
dir_path: Directory path to search in
pattern: File pattern to match (e.g. "*.py")
is_recursive: Whether to search recursively in subdirectories
- excluded_dirs: List of directories to exclude from the search
+ excluded_dirs: List of directory names to exclude from the search (e.g. [".venv", "node_modules"])
+ force_include_dirs: List of directories to force include even if they are within excluded_dirs.
+ Can be either full absolute paths or directory names.
+
Returns:
List of matching Path objects
@@ -366,6 +375,29 @@ def find_files_in_dir(dir_path: str, pattern: str, is_recursive: bool = True, ex
files = list(path.glob(pattern))
for file in files:
- if excluded_dirs is None or not any(part in excluded_dirs for part in file.parts):
+ is_excluded = excluded_dirs is not None and any(excluded_dir in file.parts for excluded_dir in excluded_dirs)
+
+ # Check if file is in a force include directory (forced inclusion despite exclusions)
+ # Force include dirs can be full paths or directory names
+ should_force_include = False
+ if force_include_dirs is not None:
+ for force_include_dir in force_include_dirs:
+ force_include_path = Path(force_include_dir)
+ # If force_include_dir is an absolute path, check if file is under it
+ if force_include_path.is_absolute():
+ try:
+ file.relative_to(force_include_path)
+ should_force_include = True
+ break
+ except ValueError:
+ # file is not relative to force_include_path
+ continue
+ # If force_include_dir is just a directory name, check if it's in the file path parts
+ elif force_include_dir in file.parts:
+ should_force_include = True
+ break
+
+ # Include if not excluded, or if force included
+ if not is_excluded or should_force_include:
filtered_files.append(file)
return filtered_files
diff --git a/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py b/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py
index eee007216..67725a2c2 100644
--- a/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py
+++ b/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py
@@ -64,3 +64,386 @@ class TestFindFilesInDir:
files = find_files_in_dir(temp_dir, "*.py", is_recursive=False)
assert len(files) == 0
+
+ def test_find_files_with_excluded_dirs_single(self):
+ """Test finding files with a single excluded directory."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Create files in root
+ (Path(temp_dir) / "root.py").touch()
+
+ # Create excluded directory
+ excluded_dir = Path(temp_dir) / ".venv"
+ excluded_dir.mkdir()
+ (excluded_dir / "excluded.py").touch()
+
+ # Create nested structure in excluded dir
+ nested_excluded = excluded_dir / "lib" / "python3.11"
+ nested_excluded.mkdir(parents=True)
+ (nested_excluded / "nested_excluded.py").touch()
+
+ # Create normal subdirectory
+ normal_dir = Path(temp_dir) / "src"
+ normal_dir.mkdir()
+ (normal_dir / "normal.py").touch()
+
+ files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=[".venv"])
+
+ assert len(files) == 2
+ file_names = [f.name for f in files]
+ assert "root.py" in file_names
+ assert "normal.py" in file_names
+ assert "excluded.py" not in file_names
+ assert "nested_excluded.py" not in file_names
+
+ def test_find_files_with_excluded_dirs_multiple(self):
+ """Test finding files with multiple excluded directories."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Create files in root
+ (Path(temp_dir) / "root.py").touch()
+
+ # Create multiple excluded directories
+ venv_dir = Path(temp_dir) / ".venv"
+ venv_dir.mkdir()
+ (venv_dir / "venv_file.py").touch()
+
+ node_modules = Path(temp_dir) / "node_modules"
+ node_modules.mkdir()
+ (node_modules / "node_file.py").touch()
+
+ pycache = Path(temp_dir) / "src" / "__pycache__"
+ pycache.mkdir(parents=True)
+ (pycache / "cache_file.py").touch()
+
+ # Create normal subdirectory
+ src_dir = Path(temp_dir) / "src"
+ (src_dir / "normal.py").touch()
+
+ files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=[".venv", "node_modules", "__pycache__"])
+
+ assert len(files) == 2
+ file_names = [f.name for f in files]
+ assert "root.py" in file_names
+ assert "normal.py" in file_names
+ assert "venv_file.py" not in file_names
+ assert "node_file.py" not in file_names
+ assert "cache_file.py" not in file_names
+
+ def test_find_files_with_force_include_dirs_full_path(self):
+ """Test finding files with force force include directories using full paths."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Create files in root
+ (Path(temp_dir) / "root.py").touch()
+
+ # Create excluded directory structure
+ venv_dir = Path(temp_dir) / ".venv"
+ venv_lib = venv_dir / "lib" / "python3.11" / "site-packages"
+ venv_lib.mkdir(parents=True)
+ (venv_lib / "excluded.py").touch()
+
+ # Create include directory within excluded
+ force_include_dir = venv_lib / "pipelex"
+ force_include_dir.mkdir()
+ (force_include_dir / "important.py").touch()
+
+ # Create nested file in include dir
+ include_nested = force_include_dir / "builder"
+ include_nested.mkdir()
+ (include_nested / "builder.py").touch()
+
+ # Create another file in venv but outside include dir
+ other_venv = venv_lib / "other_package"
+ other_venv.mkdir()
+ (other_venv / "other.py").touch()
+
+ files = find_files_in_dir(
+ temp_dir,
+ "*.py",
+ is_recursive=True,
+ excluded_dirs=[".venv"],
+ force_include_dirs=[str(force_include_dir)],
+ )
+
+ file_names = [f.name for f in files]
+ file_paths_str = [str(f) for f in files]
+
+ # Should include root and include dir files, but not other venv files
+ assert len(files) == 3
+ assert "root.py" in file_names
+ assert "important.py" in file_names
+ assert "builder.py" in file_names
+ assert "excluded.py" not in file_names
+ assert "other.py" not in file_names
+
+ # Verify the full paths are correct
+ assert any(str(force_include_dir / "important.py") in path for path in file_paths_str)
+ assert any(str(include_nested / "builder.py") in path for path in file_paths_str)
+
+ def test_find_files_with_force_include_dirs_directory_name(self):
+ """Test finding files with force force include directories using directory names."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Create files in root
+ (Path(temp_dir) / "root.py").touch()
+
+ # Create excluded directory structure
+ venv_dir = Path(temp_dir) / ".venv"
+ venv_lib = venv_dir / "lib"
+ venv_lib.mkdir(parents=True)
+ (venv_lib / "excluded.py").touch()
+
+ # Create include directory by name within excluded
+ pipelex_dir = venv_lib / "pipelex"
+ pipelex_dir.mkdir()
+ (pipelex_dir / "important.py").touch()
+
+ # Create another pipelex directory elsewhere to test name matching
+ src_pipelex = Path(temp_dir) / "src" / "pipelex"
+ src_pipelex.mkdir(parents=True)
+ (src_pipelex / "also_important.py").touch()
+
+ files = find_files_in_dir(
+ temp_dir,
+ "*.py",
+ is_recursive=True,
+ excluded_dirs=[".venv"],
+ force_include_dirs=["pipelex"],
+ )
+
+ file_names = [f.name for f in files]
+
+ # Should include root, both pipelex directories (one from venv as included, one from src)
+ assert len(files) == 3
+ assert "root.py" in file_names
+ assert "important.py" in file_names
+ assert "also_important.py" in file_names
+ assert "excluded.py" not in file_names
+
+ def test_find_files_with_force_include_dirs_mixed_types(self):
+ """Test finding files with force force include directories using both full paths and names."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Create files in root
+ (Path(temp_dir) / "root.py").touch()
+
+ # Create excluded directory structure
+ venv_dir = Path(temp_dir) / ".venv"
+ venv_packages = venv_dir / "lib" / "site-packages"
+ venv_packages.mkdir(parents=True)
+
+ # Include by full path
+ pipelex_dir = venv_packages / "pipelex"
+ pipelex_dir.mkdir()
+ (pipelex_dir / "pipelex_file.py").touch()
+
+ # Another excluded directory
+ node_modules = Path(temp_dir) / "node_modules"
+ node_modules.mkdir()
+ (node_modules / "node_file.py").touch()
+
+ # Include by name within node_modules
+ special_dir = node_modules / "special"
+ special_dir.mkdir()
+ (special_dir / "special_file.py").touch()
+
+ # Regular file in node_modules
+ (node_modules / "regular_node.py").touch()
+
+ files = find_files_in_dir(
+ temp_dir,
+ "*.py",
+ is_recursive=True,
+ excluded_dirs=[".venv", "node_modules"],
+ force_include_dirs=[str(pipelex_dir), "special"],
+ )
+
+ file_names = [f.name for f in files]
+
+ # Should include root, pipelex (full path include), and special (name include)
+ assert len(files) == 3
+ assert "root.py" in file_names
+ assert "pipelex_file.py" in file_names
+ assert "special_file.py" in file_names
+ assert "node_file.py" not in file_names
+ assert "regular_node.py" not in file_names
+
+ def test_find_files_with_multiple_force_include_dirs_full_paths(self):
+ """Test finding files with multiple force include directories using full paths."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Create files in root
+ (Path(temp_dir) / "root.py").touch()
+
+ # Create excluded directory
+ venv_dir = Path(temp_dir) / ".venv"
+ packages = venv_dir / "site-packages"
+ packages.mkdir(parents=True)
+
+ # First include directory
+ pkg1 = packages / "package1"
+ pkg1.mkdir()
+ (pkg1 / "pkg1_file.py").touch()
+
+ # Second include directory
+ pkg2 = packages / "package2"
+ pkg2.mkdir()
+ (pkg2 / "pkg2_file.py").touch()
+
+ # Non-included directory
+ pkg3 = packages / "package3"
+ pkg3.mkdir()
+ (pkg3 / "pkg3_file.py").touch()
+
+ files = find_files_in_dir(
+ temp_dir,
+ "*.py",
+ is_recursive=True,
+ excluded_dirs=[".venv"],
+ force_include_dirs=[str(pkg1), str(pkg2)],
+ )
+
+ file_names = [f.name for f in files]
+
+ assert len(files) == 3
+ assert "root.py" in file_names
+ assert "pkg1_file.py" in file_names
+ assert "pkg2_file.py" in file_names
+ assert "pkg3_file.py" not in file_names
+
+ def test_find_files_with_deeply_nested_force_include(self):
+ """Test finding files with deeply nested force include directories."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Create deeply nested structure
+ deep_path = Path(temp_dir) / ".venv" / "lib" / "python3.11" / "site-packages" / "pipelex" / "builder"
+ deep_path.mkdir(parents=True)
+ (deep_path / "deep_file.py").touch()
+
+ # Create file at intermediate level that should be excluded
+ intermediate = Path(temp_dir) / ".venv" / "lib" / "python3.11" / "site-packages"
+ (intermediate / "intermediate.py").touch()
+
+ # Create file at root
+ (Path(temp_dir) / "root.py").touch()
+
+ files = find_files_in_dir(
+ temp_dir,
+ "*.py",
+ is_recursive=True,
+ excluded_dirs=[".venv"],
+ force_include_dirs=[str(deep_path)],
+ )
+
+ file_names = [f.name for f in files]
+
+ assert len(files) == 2
+ assert "root.py" in file_names
+ assert "deep_file.py" in file_names
+ assert "intermediate.py" not in file_names
+
+ def test_find_files_no_excluded_dirs(self):
+ """Test that passing None for excluded_dirs works correctly."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Create files
+ (Path(temp_dir) / "file1.py").touch()
+
+ venv_dir = Path(temp_dir) / ".venv"
+ venv_dir.mkdir()
+ (venv_dir / "venv_file.py").touch()
+
+ files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=None)
+
+ # Should find all files when no exclusions
+ assert len(files) == 2
+ file_names = [f.name for f in files]
+ assert "file1.py" in file_names
+ assert "venv_file.py" in file_names
+
+ def test_find_files_no_force_include_dirs(self):
+ """Test that passing None for force_include_dirs works correctly."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Create files
+ (Path(temp_dir) / "file1.py").touch()
+
+ venv_dir = Path(temp_dir) / ".venv"
+ venv_dir.mkdir()
+ (venv_dir / "venv_file.py").touch()
+
+ files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=[".venv"], force_include_dirs=None)
+
+ # Should exclude all venv files when no includes
+ assert len(files) == 1
+ assert files[0].name == "file1.py"
+
+ def test_find_files_force_include_not_in_excluded_dir(self):
+ """Test that force include directories outside excluded directories are handled correctly."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Create normal directory
+ src_dir = Path(temp_dir) / "src"
+ src_dir.mkdir()
+ (src_dir / "src_file.py").touch()
+
+ # Create excluded directory
+ venv_dir = Path(temp_dir) / ".venv"
+ venv_dir.mkdir()
+ (venv_dir / "venv_file.py").touch()
+
+ # Include directory is in src, not in excluded dir
+ files = find_files_in_dir(
+ temp_dir,
+ "*.py",
+ is_recursive=True,
+ excluded_dirs=[".venv"],
+ force_include_dirs=[str(src_dir)],
+ )
+
+ # Should find src file (not excluded) but not venv file
+ assert len(files) == 1
+ assert files[0].name == "src_file.py"
+
+ def test_find_files_pattern_matching(self):
+ """Test that file pattern matching works correctly with exclusions."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ # Create various file types
+ (Path(temp_dir) / "script.py").touch()
+ (Path(temp_dir) / "data.json").touch()
+ (Path(temp_dir) / "config.toml").touch()
+
+ venv_dir = Path(temp_dir) / ".venv"
+ venv_dir.mkdir()
+ (venv_dir / "venv_script.py").touch()
+ (venv_dir / "venv_data.json").touch()
+
+ # Find only .py files
+ py_files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=[".venv"])
+ assert len(py_files) == 1
+ assert py_files[0].name == "script.py"
+
+ # Find only .json files
+ json_files = find_files_in_dir(temp_dir, "*.json", is_recursive=True, excluded_dirs=[".venv"])
+ assert len(json_files) == 1
+ assert json_files[0].name == "data.json"
+
+ def test_find_files_empty_excluded_list(self):
+ """Test that passing an empty list for excluded_dirs works correctly."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ (Path(temp_dir) / "file1.py").touch()
+
+ venv_dir = Path(temp_dir) / ".venv"
+ venv_dir.mkdir()
+ (venv_dir / "venv_file.py").touch()
+
+ files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=[])
+
+ # Empty list should behave like None - no exclusions
+ assert len(files) == 2
+
+ def test_find_files_empty_force_include_list(self):
+ """Test that passing an empty list for force_include_dirs works correctly."""
+ with tempfile.TemporaryDirectory() as temp_dir:
+ (Path(temp_dir) / "file1.py").touch()
+
+ venv_dir = Path(temp_dir) / ".venv"
+ venv_dir.mkdir()
+ (venv_dir / "venv_file.py").touch()
+
+ files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=[".venv"], force_include_dirs=[])
+
+ # Empty include list should exclude all venv files
+ assert len(files) == 1
+ assert files[0].name == "file1.py"