Build AI methods with your coding agent, then run them anywhere: as an MCP for chatbots, as a webapp for people, or via API for your software.
fix/excluded-dirs-pipe-builder (#495)
Thomas Hebrard85d5fa49b8ca
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 4ebd06c3a..859ed1730 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -1,5 +1,11 @@# Changelog+### Unreleased++### Added++- Added `force_include_dirs` parameter to the `find_files_in_dir` function. This allows specific directories to be force included in the search even when they are nested within excluded directories. For example, you can now exclude `.venv` while still including `.venv/lib/python3.11/site-packages/pipelex` for loading Pipelex libraries from installed packages.+## [v0.17.1] - 2025-11-27### Fixeddiff --git a/pipelex/libraries/library_utils.py b/pipelex/libraries/library_utils.pyindex 7cae920b9..5d8d1ec4a 100644--- a/pipelex/libraries/library_utils.py+++ b/pipelex/libraries/library_utils.py@@ -77,6 +77,7 @@ def get_pipelex_plx_files_from_dirs(dirs: set[Path]) -> list[Path]:dir_path=str(dir_path),pattern="*.plx",excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),+ force_include_dirs=[str(dir_path) for dir_path in dirs],)# Filter to only include valid Pipelex filesdiff --git a/pipelex/system/registries/class_registry_utils.py b/pipelex/system/registries/class_registry_utils.pyindex 2b0dc2eb4..474615597 100644--- a/pipelex/system/registries/class_registry_utils.py+++ b/pipelex/system/registries/class_registry_utils.py@@ -97,6 +97,7 @@ class ClassRegistryUtils:pattern="*.py",is_recursive=is_recursive,excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),+ force_include_dirs=[folder_path],)for python_file in python_files:diff --git a/pipelex/system/registries/func_registry_utils.py b/pipelex/system/registries/func_registry_utils.pyindex f71b2e9cd..668920e91 100644--- a/pipelex/system/registries/func_registry_utils.py+++ b/pipelex/system/registries/func_registry_utils.py@@ -87,6 +87,7 @@ class FuncRegistryUtils:pattern="*.py",is_recursive=is_recursive,excluded_dirs=list(get_config().pipelex.scan_config.excluded_dirs),+ force_include_dirs=[folder_path],)for python_file in python_files:diff --git a/pipelex/tools/misc/file_utils.py b/pipelex/tools/misc/file_utils.pyindex 281b7c839..7e67ffb11 100644--- a/pipelex/tools/misc/file_utils.py+++ b/pipelex/tools/misc/file_utils.py@@ -345,14 +345,23 @@ def get_incremental_file_path(########################################################-def find_files_in_dir(dir_path: str, pattern: str, is_recursive: bool = True, excluded_dirs: list[str] | None = None) -> list[Path]:+def find_files_in_dir(+ dir_path: str,+ pattern: str,+ is_recursive: bool = True,+ excluded_dirs: list[str] | None = None,+ force_include_dirs: list[str] | None = None,+) -> list[Path]:"""Find files matching a pattern in a directory.Args:dir_path: Directory path to search inpattern: File pattern to match (e.g. "*.py")is_recursive: Whether to search recursively in subdirectories- excluded_dirs: List of directories to exclude from the search+ excluded_dirs: List of directory names to exclude from the search (e.g. [".venv", "node_modules"])+ force_include_dirs: List of directories to force include even if they are within excluded_dirs.+ Can be either full absolute paths or directory names.+Returns:List of matching Path objects@@ -366,6 +375,29 @@ def find_files_in_dir(dir_path: str, pattern: str, is_recursive: bool = True, exfiles = list(path.glob(pattern))for file in files:- if excluded_dirs is None or not any(part in excluded_dirs for part in file.parts):+ is_excluded = excluded_dirs is not None and any(excluded_dir in file.parts for excluded_dir in excluded_dirs)++ # Check if file is in a force include directory (forced inclusion despite exclusions)+ # Force include dirs can be full paths or directory names+ should_force_include = False+ if force_include_dirs is not None:+ for force_include_dir in force_include_dirs:+ force_include_path = Path(force_include_dir)+ # If force_include_dir is an absolute path, check if file is under it+ if force_include_path.is_absolute():+ try:+ file.relative_to(force_include_path)+ should_force_include = True+ break+ except ValueError:+ # file is not relative to force_include_path+ continue+ # If force_include_dir is just a directory name, check if it's in the file path parts+ elif force_include_dir in file.parts:+ should_force_include = True+ break++ # Include if not excluded, or if force included+ if not is_excluded or should_force_include:filtered_files.append(file)return filtered_filesdiff --git a/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py b/tests/unit/pipelex/tools/misc/test_find_files_in_dir.pyindex eee007216..67725a2c2 100644--- a/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py+++ b/tests/unit/pipelex/tools/misc/test_find_files_in_dir.py@@ -64,3 +64,386 @@ class TestFindFilesInDir:files = find_files_in_dir(temp_dir, "*.py", is_recursive=False)assert len(files) == 0++ def test_find_files_with_excluded_dirs_single(self):+ """Test finding files with a single excluded directory."""+ with tempfile.TemporaryDirectory() as temp_dir:+ # Create files in root+ (Path(temp_dir) / "root.py").touch()++ # Create excluded directory+ excluded_dir = Path(temp_dir) / ".venv"+ excluded_dir.mkdir()+ (excluded_dir / "excluded.py").touch()++ # Create nested structure in excluded dir+ nested_excluded = excluded_dir / "lib" / "python3.11"+ nested_excluded.mkdir(parents=True)+ (nested_excluded / "nested_excluded.py").touch()++ # Create normal subdirectory+ normal_dir = Path(temp_dir) / "src"+ normal_dir.mkdir()+ (normal_dir / "normal.py").touch()++ files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=[".venv"])++ assert len(files) == 2+ file_names = [f.name for f in files]+ assert "root.py" in file_names+ assert "normal.py" in file_names+ assert "excluded.py" not in file_names+ assert "nested_excluded.py" not in file_names++ def test_find_files_with_excluded_dirs_multiple(self):+ """Test finding files with multiple excluded directories."""+ with tempfile.TemporaryDirectory() as temp_dir:+ # Create files in root+ (Path(temp_dir) / "root.py").touch()++ # Create multiple excluded directories+ venv_dir = Path(temp_dir) / ".venv"+ venv_dir.mkdir()+ (venv_dir / "venv_file.py").touch()++ node_modules = Path(temp_dir) / "node_modules"+ node_modules.mkdir()+ (node_modules / "node_file.py").touch()++ pycache = Path(temp_dir) / "src" / "__pycache__"+ pycache.mkdir(parents=True)+ (pycache / "cache_file.py").touch()++ # Create normal subdirectory+ src_dir = Path(temp_dir) / "src"+ (src_dir / "normal.py").touch()++ files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=[".venv", "node_modules", "__pycache__"])++ assert len(files) == 2+ file_names = [f.name for f in files]+ assert "root.py" in file_names+ assert "normal.py" in file_names+ assert "venv_file.py" not in file_names+ assert "node_file.py" not in file_names+ assert "cache_file.py" not in file_names++ def test_find_files_with_force_include_dirs_full_path(self):+ """Test finding files with force force include directories using full paths."""+ with tempfile.TemporaryDirectory() as temp_dir:+ # Create files in root+ (Path(temp_dir) / "root.py").touch()++ # Create excluded directory structure+ venv_dir = Path(temp_dir) / ".venv"+ venv_lib = venv_dir / "lib" / "python3.11" / "site-packages"+ venv_lib.mkdir(parents=True)+ (venv_lib / "excluded.py").touch()++ # Create include directory within excluded+ force_include_dir = venv_lib / "pipelex"+ force_include_dir.mkdir()+ (force_include_dir / "important.py").touch()++ # Create nested file in include dir+ include_nested = force_include_dir / "builder"+ include_nested.mkdir()+ (include_nested / "builder.py").touch()++ # Create another file in venv but outside include dir+ other_venv = venv_lib / "other_package"+ other_venv.mkdir()+ (other_venv / "other.py").touch()++ files = find_files_in_dir(+ temp_dir,+ "*.py",+ is_recursive=True,+ excluded_dirs=[".venv"],+ force_include_dirs=[str(force_include_dir)],+ )++ file_names = [f.name for f in files]+ file_paths_str = [str(f) for f in files]++ # Should include root and include dir files, but not other venv files+ assert len(files) == 3+ assert "root.py" in file_names+ assert "important.py" in file_names+ assert "builder.py" in file_names+ assert "excluded.py" not in file_names+ assert "other.py" not in file_names++ # Verify the full paths are correct+ assert any(str(force_include_dir / "important.py") in path for path in file_paths_str)+ assert any(str(include_nested / "builder.py") in path for path in file_paths_str)++ def test_find_files_with_force_include_dirs_directory_name(self):+ """Test finding files with force force include directories using directory names."""+ with tempfile.TemporaryDirectory() as temp_dir:+ # Create files in root+ (Path(temp_dir) / "root.py").touch()++ # Create excluded directory structure+ venv_dir = Path(temp_dir) / ".venv"+ venv_lib = venv_dir / "lib"+ venv_lib.mkdir(parents=True)+ (venv_lib / "excluded.py").touch()++ # Create include directory by name within excluded+ pipelex_dir = venv_lib / "pipelex"+ pipelex_dir.mkdir()+ (pipelex_dir / "important.py").touch()++ # Create another pipelex directory elsewhere to test name matching+ src_pipelex = Path(temp_dir) / "src" / "pipelex"+ src_pipelex.mkdir(parents=True)+ (src_pipelex / "also_important.py").touch()++ files = find_files_in_dir(+ temp_dir,+ "*.py",+ is_recursive=True,+ excluded_dirs=[".venv"],+ force_include_dirs=["pipelex"],+ )++ file_names = [f.name for f in files]++ # Should include root, both pipelex directories (one from venv as included, one from src)+ assert len(files) == 3+ assert "root.py" in file_names+ assert "important.py" in file_names+ assert "also_important.py" in file_names+ assert "excluded.py" not in file_names++ def test_find_files_with_force_include_dirs_mixed_types(self):+ """Test finding files with force force include directories using both full paths and names."""+ with tempfile.TemporaryDirectory() as temp_dir:+ # Create files in root+ (Path(temp_dir) / "root.py").touch()++ # Create excluded directory structure+ venv_dir = Path(temp_dir) / ".venv"+ venv_packages = venv_dir / "lib" / "site-packages"+ venv_packages.mkdir(parents=True)++ # Include by full path+ pipelex_dir = venv_packages / "pipelex"+ pipelex_dir.mkdir()+ (pipelex_dir / "pipelex_file.py").touch()++ # Another excluded directory+ node_modules = Path(temp_dir) / "node_modules"+ node_modules.mkdir()+ (node_modules / "node_file.py").touch()++ # Include by name within node_modules+ special_dir = node_modules / "special"+ special_dir.mkdir()+ (special_dir / "special_file.py").touch()++ # Regular file in node_modules+ (node_modules / "regular_node.py").touch()++ files = find_files_in_dir(+ temp_dir,+ "*.py",+ is_recursive=True,+ excluded_dirs=[".venv", "node_modules"],+ force_include_dirs=[str(pipelex_dir), "special"],+ )++ file_names = [f.name for f in files]++ # Should include root, pipelex (full path include), and special (name include)+ assert len(files) == 3+ assert "root.py" in file_names+ assert "pipelex_file.py" in file_names+ assert "special_file.py" in file_names+ assert "node_file.py" not in file_names+ assert "regular_node.py" not in file_names++ def test_find_files_with_multiple_force_include_dirs_full_paths(self):+ """Test finding files with multiple force include directories using full paths."""+ with tempfile.TemporaryDirectory() as temp_dir:+ # Create files in root+ (Path(temp_dir) / "root.py").touch()++ # Create excluded directory+ venv_dir = Path(temp_dir) / ".venv"+ packages = venv_dir / "site-packages"+ packages.mkdir(parents=True)++ # First include directory+ pkg1 = packages / "package1"+ pkg1.mkdir()+ (pkg1 / "pkg1_file.py").touch()++ # Second include directory+ pkg2 = packages / "package2"+ pkg2.mkdir()+ (pkg2 / "pkg2_file.py").touch()++ # Non-included directory+ pkg3 = packages / "package3"+ pkg3.mkdir()+ (pkg3 / "pkg3_file.py").touch()++ files = find_files_in_dir(+ temp_dir,+ "*.py",+ is_recursive=True,+ excluded_dirs=[".venv"],+ force_include_dirs=[str(pkg1), str(pkg2)],+ )++ file_names = [f.name for f in files]++ assert len(files) == 3+ assert "root.py" in file_names+ assert "pkg1_file.py" in file_names+ assert "pkg2_file.py" in file_names+ assert "pkg3_file.py" not in file_names++ def test_find_files_with_deeply_nested_force_include(self):+ """Test finding files with deeply nested force include directories."""+ with tempfile.TemporaryDirectory() as temp_dir:+ # Create deeply nested structure+ deep_path = Path(temp_dir) / ".venv" / "lib" / "python3.11" / "site-packages" / "pipelex" / "builder"+ deep_path.mkdir(parents=True)+ (deep_path / "deep_file.py").touch()++ # Create file at intermediate level that should be excluded+ intermediate = Path(temp_dir) / ".venv" / "lib" / "python3.11" / "site-packages"+ (intermediate / "intermediate.py").touch()++ # Create file at root+ (Path(temp_dir) / "root.py").touch()++ files = find_files_in_dir(+ temp_dir,+ "*.py",+ is_recursive=True,+ excluded_dirs=[".venv"],+ force_include_dirs=[str(deep_path)],+ )++ file_names = [f.name for f in files]++ assert len(files) == 2+ assert "root.py" in file_names+ assert "deep_file.py" in file_names+ assert "intermediate.py" not in file_names++ def test_find_files_no_excluded_dirs(self):+ """Test that passing None for excluded_dirs works correctly."""+ with tempfile.TemporaryDirectory() as temp_dir:+ # Create files+ (Path(temp_dir) / "file1.py").touch()++ venv_dir = Path(temp_dir) / ".venv"+ venv_dir.mkdir()+ (venv_dir / "venv_file.py").touch()++ files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=None)++ # Should find all files when no exclusions+ assert len(files) == 2+ file_names = [f.name for f in files]+ assert "file1.py" in file_names+ assert "venv_file.py" in file_names++ def test_find_files_no_force_include_dirs(self):+ """Test that passing None for force_include_dirs works correctly."""+ with tempfile.TemporaryDirectory() as temp_dir:+ # Create files+ (Path(temp_dir) / "file1.py").touch()++ venv_dir = Path(temp_dir) / ".venv"+ venv_dir.mkdir()+ (venv_dir / "venv_file.py").touch()++ files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=[".venv"], force_include_dirs=None)++ # Should exclude all venv files when no includes+ assert len(files) == 1+ assert files[0].name == "file1.py"++ def test_find_files_force_include_not_in_excluded_dir(self):+ """Test that force include directories outside excluded directories are handled correctly."""+ with tempfile.TemporaryDirectory() as temp_dir:+ # Create normal directory+ src_dir = Path(temp_dir) / "src"+ src_dir.mkdir()+ (src_dir / "src_file.py").touch()++ # Create excluded directory+ venv_dir = Path(temp_dir) / ".venv"+ venv_dir.mkdir()+ (venv_dir / "venv_file.py").touch()++ # Include directory is in src, not in excluded dir+ files = find_files_in_dir(+ temp_dir,+ "*.py",+ is_recursive=True,+ excluded_dirs=[".venv"],+ force_include_dirs=[str(src_dir)],+ )++ # Should find src file (not excluded) but not venv file+ assert len(files) == 1+ assert files[0].name == "src_file.py"++ def test_find_files_pattern_matching(self):+ """Test that file pattern matching works correctly with exclusions."""+ with tempfile.TemporaryDirectory() as temp_dir:+ # Create various file types+ (Path(temp_dir) / "script.py").touch()+ (Path(temp_dir) / "data.json").touch()+ (Path(temp_dir) / "config.toml").touch()++ venv_dir = Path(temp_dir) / ".venv"+ venv_dir.mkdir()+ (venv_dir / "venv_script.py").touch()+ (venv_dir / "venv_data.json").touch()++ # Find only .py files+ py_files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=[".venv"])+ assert len(py_files) == 1+ assert py_files[0].name == "script.py"++ # Find only .json files+ json_files = find_files_in_dir(temp_dir, "*.json", is_recursive=True, excluded_dirs=[".venv"])+ assert len(json_files) == 1+ assert json_files[0].name == "data.json"++ def test_find_files_empty_excluded_list(self):+ """Test that passing an empty list for excluded_dirs works correctly."""+ with tempfile.TemporaryDirectory() as temp_dir:+ (Path(temp_dir) / "file1.py").touch()++ venv_dir = Path(temp_dir) / ".venv"+ venv_dir.mkdir()+ (venv_dir / "venv_file.py").touch()++ files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=[])++ # Empty list should behave like None - no exclusions+ assert len(files) == 2++ def test_find_files_empty_force_include_list(self):+ """Test that passing an empty list for force_include_dirs works correctly."""+ with tempfile.TemporaryDirectory() as temp_dir:+ (Path(temp_dir) / "file1.py").touch()++ venv_dir = Path(temp_dir) / ".venv"+ venv_dir.mkdir()+ (venv_dir / "venv_file.py").touch()++ files = find_files_in_dir(temp_dir, "*.py", is_recursive=True, excluded_dirs=[".venv"], force_include_dirs=[])++ # Empty include list should exclude all venv files+ assert len(files) == 1+ assert files[0].name == "file1.py"