Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.
TypeScript agentic-aiagentsaiai-agentsaiagentsdeveloper-toolsfunction-callinggpt-4javascriptjsllmllmopsmcppythonremote-mcp-serverssetypescript
fix(python): create the cache directory on first use instead of at import time (#4162)
Alberto Schiabel03429c74044b
diff --git a/python/composio/core/models/_files.py b/python/composio/core/models/_files.pyindex dd3c5e84e..cd3dcbd5a 100644--- a/python/composio/core/models/_files.py+++ b/python/composio/core/models/_files.py@@ -82,35 +82,85 @@ ENV_LOCAL_CACHE_DIRECTORY = "COMPOSIO_CACHE_DIR"Environment to set the composio caching directory."""-LOCAL_CACHE_DIRECTORY = Path(- os.environ.get(- ENV_LOCAL_CACHE_DIRECTORY,- Path.home() / LOCAL_CACHE_DIRECTORY_NAME, # Fallback to user directory- )-)+LOCAL_OUTPUT_FILE_DIRECTORY_NAME = "files""""-Path to local caching directory.+Name of the cache sub-directory into which files downloaded during tool+execution are written. Previously ``outputs``; now ``files`` for parity with+the TypeScript SDK."""-try:- LOCAL_CACHE_DIRECTORY.mkdir(parents=True, exist_ok=True)- if not os.access(LOCAL_CACHE_DIRECTORY, os.W_OK):- raise OSError-except OSError as e:- raise RuntimeError(- f"Cache directory {LOCAL_CACHE_DIRECTORY} is not writable please "- f"provide a path that is writable using {ENV_LOCAL_CACHE_DIRECTORY} "- "environment variable."- ) from e+def get_cache_directory() -> Path:+ """Resolve the local caching directory without touching the filesystem.-LOCAL_OUTPUT_FILE_DIRECTORY = LOCAL_CACHE_DIRECTORY / "files"-"""-Default local directory into which files downloaded during tool execution are-written. Previously ``<cache>/outputs``; now ``<cache>/files`` for parity with-the TypeScript SDK. Override by passing ``file_download_dir=...`` to Composio,-or by setting ``outdir`` on ``FileHelper`` directly.-"""+ ``COMPOSIO_CACHE_DIR`` is read on every call, so it can be set after+ ``composio`` has already been imported. ``Path.home()`` is only consulted+ when the variable is unset: it can raise ``RuntimeError`` when there is no+ resolvable home directory, which is exactly the situation+ ``COMPOSIO_CACHE_DIR`` exists to work around, so it must not be evaluated+ eagerly as a fallback argument.+ """+ configured = os.environ.get(ENV_LOCAL_CACHE_DIRECTORY)+ if configured:+ return Path(configured)++ try:+ home = Path.home()+ except RuntimeError as e:+ raise RuntimeError(+ "Could not determine a home directory to store the Composio cache "+ f"in. Provide a writable path using the {ENV_LOCAL_CACHE_DIRECTORY} "+ "environment variable."+ ) from e+ return home / LOCAL_CACHE_DIRECTORY_NAME+++def get_output_file_directory() -> Path:+ """Default local directory into which files downloaded during tool+ execution are written. Override by passing ``file_download_dir=...`` to+ Composio, or by setting ``outdir`` on ``FileHelper`` directly.+ """+ return get_cache_directory() / LOCAL_OUTPUT_FILE_DIRECTORY_NAME+++def ensure_cache_directory() -> Path:+ """Create the cache directory on first use and check that it is writable.++ This used to run at module import time, so a bare ``import composio``+ raised ``RuntimeError`` on any read-only filesystem -- AWS Lambda,+ distroless containers, ``ProtectHome=true`` systemd units -- even for+ programs that never touched a file. Deferring it to first use keeps the+ same check, and the same error message, for the callers that actually+ need the directory.+ """+ directory = get_cache_directory()+ try:+ directory.mkdir(parents=True, exist_ok=True)+ if not os.access(directory, os.W_OK):+ raise OSError+ except OSError as e:+ raise RuntimeError(+ f"Cache directory {directory} is not writable please "+ f"provide a path that is writable using {ENV_LOCAL_CACHE_DIRECTORY} "+ "environment variable."+ ) from e+ return directory+++def __getattr__(name: str) -> Path:+ """Keep the historical module-level path constants working, but lazily.++ ``LOCAL_CACHE_DIRECTORY`` and ``LOCAL_OUTPUT_FILE_DIRECTORY`` used to be+ computed at import time. They are now resolved on attribute access+ instead (PEP 562), so importing this module no longer touches the+ filesystem or depends on the environment, and both constants observe a+ ``COMPOSIO_CACHE_DIR`` that was set after import.+ """+ if name == "LOCAL_CACHE_DIRECTORY":+ return get_cache_directory()+ if name == "LOCAL_OUTPUT_FILE_DIRECTORY":+ return get_output_file_directory()+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")def get_md5(file: Path) -> str:@@ -670,7 +720,11 @@ class FileHelper(WithLogger):"""super().__init__()self._client = client- self._outdir = Path(outdir) if outdir else LOCAL_OUTPUT_FILE_DIRECTORY+ # Falsy, not just `is None`: an empty string falls through to the+ # same default directory below, and must be treated as such here too+ # or `ensure_cache_directory()` silently stops firing for it.+ self._outdir_is_default = not outdir+ self._outdir = Path(outdir) if outdir else get_output_file_directory()self._sensitive_file_upload_protection = sensitive_file_upload_protectionself._file_upload_path_deny_segments = file_upload_path_deny_segmentsself._file_upload_allowlist: t.Optional[t.Sequence[Path]] = (@@ -1103,6 +1157,11 @@ class FileHelper(WithLogger):def _download_file_value(self, value: t.Any, tool: Tool) -> t.Any:if isinstance(value, dict) and "s3url" in value:+ if self._outdir_is_default:+ # First point at which the cache directory is genuinely+ # needed. Raises the same RuntimeError that used to be raised+ # at import time, pointing at COMPOSIO_CACHE_DIR.+ ensure_cache_directory()# `tool.toolkit.slug` and `tool.slug` come from the API response and# are untrusted, so joining them directly would let the response pick# the download directory. `secure_join` validates each component anddiff --git a/python/tests/test_files.py b/python/tests/test_files.pyindex c4a7bd60c..ac6dda819 100644--- a/python/tests/test_files.py+++ b/python/tests/test_files.py@@ -52,8 +52,15 @@ def mock_client():@pytest.fixture-def file_helper(mock_client):- """Create a FileHelper instance with a mock client."""+def file_helper(mock_client, monkeypatch, tmp_path):+ """Create a FileHelper instance with a mock client.++ Sandboxes ``COMPOSIO_CACHE_DIR`` so the default-outdir download path+ (``ensure_cache_directory()``) never touches the real home directory,+ even though ``FileDownloadable.download`` itself is mocked in most of+ these tests.+ """+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / ".composio"))return FileHelper(client=mock_client)diff --git a/python/tests/test_lazy_cache_directory.py b/python/tests/test_lazy_cache_directory.pynew file mode 100644index 000000000..eadc5eda2--- /dev/null+++ b/python/tests/test_lazy_cache_directory.py@@ -0,0 +1,248 @@+"""Regression tests for lazy creation of the Composio cache directory.++``composio/core/models/_files.py`` used to create ``~/.composio`` and assert+that it was writable at *module import time*. Because ``composio/__init__.py``+imports ``tools.py``, which imports ``_files.py`` unconditionally, a bare+``import composio`` raised ``RuntimeError`` on any read-only filesystem --+AWS Lambda, distroless containers, ``ProtectHome=true`` systemd units -- even+for programs that never touched a file (#4150).++These tests pin the fix: importing the package performs no filesystem work,+``COMPOSIO_CACHE_DIR`` is honoured when set after import, and the writability+check still raises the same error when the directory is genuinely needed.+"""++import os+import subprocess+import sys+from pathlib import Path+from unittest.mock import Mock++import pytest++from composio.core.models import _files+from composio.core.models._files import FileHelper+from composio.exceptions import BlockedInternalUrlError+++def unusable_cache_dir(tmp_path: Path) -> Path:+ """A cache path that cannot be created, without relying on permissions.++ ``chmod``-based read-only directories are a no-op when the test suite+ runs as root (``os.access(..., W_OK)`` returns ``True`` for uid 0), which+ is common in containerized CI. Nesting the cache directory *underneath a+ regular file* makes ``mkdir`` fail with ``NotADirectoryError`` for every+ user.+ """+ blocker = tmp_path / "blocker"+ blocker.write_text("not a directory")+ return blocker / "cache"+++def test_importing_composio_does_not_create_the_cache_directory(+ unusable_cache_dir: Path,+) -> None:+ """The regression test: ``import composio`` must not touch the filesystem.++ Run in a subprocess so the import genuinely happens under the patched+ environment rather than hitting the already-imported module in this+ process.+ """+ env = dict(os.environ)+ env["COMPOSIO_CACHE_DIR"] = str(unusable_cache_dir)++ result = subprocess.run(+ [sys.executable, "-c", "import composio"],+ env=env,+ capture_output=True,+ text=True,+ timeout=120,+ )++ assert result.returncode == 0, (+ "importing composio with an uncreatable cache directory must succeed, "+ f"but it failed with:\n{result.stderr}"+ )+ assert not unusable_cache_dir.exists()+++def test_cache_directory_is_created_only_on_demand(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ cache_dir = tmp_path / "not-yet-there"+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(cache_dir))++ # Merely resolving the path must not create it.+ assert _files.get_cache_directory() == cache_dir+ assert not cache_dir.exists()++ assert _files.ensure_cache_directory() == cache_dir+ assert cache_dir.is_dir()+++def test_cache_dir_env_var_is_read_after_import(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ """``COMPOSIO_CACHE_DIR`` used to be frozen at import time."""+ first = tmp_path / "first"+ second = tmp_path / "second"++ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(first))+ assert _files.get_cache_directory() == first++ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(second))+ assert _files.get_cache_directory() == second+++def test_home_is_not_resolved_when_the_env_var_is_set(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ """The escape hatch has to work when ``Path.home()`` itself fails.++ The old code passed ``Path.home() / ...`` as the *default argument* to+ ``os.environ.get``, which Python evaluates eagerly, so setting+ ``COMPOSIO_CACHE_DIR`` did not save a user whose home directory could not+ be resolved.+ """++ def _explode() -> Path:+ raise RuntimeError("Could not determine home directory")++ monkeypatch.setattr(Path, "home", staticmethod(_explode))+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "cache"))++ assert _files.get_cache_directory() == tmp_path / "cache"+++def test_unresolvable_home_raises_a_helpful_error(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ def _explode() -> Path:+ raise RuntimeError("Could not determine home directory")++ monkeypatch.setattr(Path, "home", staticmethod(_explode))+ monkeypatch.delenv("COMPOSIO_CACHE_DIR", raising=False)++ with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):+ _files.get_cache_directory()+++def test_output_file_directory_hangs_off_the_cache_directory(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "cache"))+ assert _files.get_output_file_directory() == tmp_path / "cache" / "files"+++def test_ensure_cache_directory_still_reports_unwritable_paths(+ monkeypatch: pytest.MonkeyPatch, unusable_cache_dir: Path+) -> None:+ """The writability check is deferred, not removed."""+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable_cache_dir))++ with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):+ _files.ensure_cache_directory()+++def test_legacy_module_constants_still_resolve(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ """``LOCAL_CACHE_DIRECTORY`` was public-ish; keep it importable (PEP 562)."""+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "cache"))++ assert _files.LOCAL_CACHE_DIRECTORY == tmp_path / "cache"+ assert _files.LOCAL_OUTPUT_FILE_DIRECTORY == tmp_path / "cache" / "files"++ # And unlike the old module-level constants, they track the environment.+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "other"))+ assert _files.LOCAL_CACHE_DIRECTORY == tmp_path / "other"+++def test_unknown_module_attribute_still_raises_attribute_error() -> None:+ """The module ``__getattr__`` must not swallow genuine typos."""+ with pytest.raises(AttributeError, match="no attribute"):+ _files.LOCAL_CACHE_DIRECTROY # noqa: B018 (deliberate typo)+++def test_file_helper_with_explicit_outdir_never_touches_the_default_cache(+ monkeypatch: pytest.MonkeyPatch, unusable_cache_dir: Path, tmp_path: Path+) -> None:+ """A caller-supplied ``outdir`` must not force the default cache directory+ into existence, and downloads into it must not trigger the writability+ check for the (unused) default cache directory."""+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable_cache_dir))++ outdir = tmp_path / "custom-outdir"+ helper = FileHelper(client=Mock(), outdir=str(outdir))+ assert helper._outdir_is_default is False++ tool = Mock()+ tool.toolkit.slug = "gmail"+ tool.slug = "GMAIL_FETCH"++ # The download itself fails fast on the blank s3url, but the point of+ # this test is what happened *before* that: the default cache directory+ # must never have been created.+ with pytest.raises(BlockedInternalUrlError):+ helper._download_file_value(+ {"name": "irrelevant.txt", "mimetype": "text/plain", "s3url": ""},+ tool,+ )+ assert not unusable_cache_dir.exists()+++def test_file_helper_default_outdir_ensures_cache_directory_on_download(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ """The default-outdir path is exactly where the deferred writability+ check must still fire, with the original error message."""+ unusable = tmp_path / "blocker-file"+ unusable.write_text("not a directory")+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable / "cache"))++ helper = FileHelper(client=Mock())+ assert helper._outdir_is_default is True++ tool = Mock()+ tool.toolkit.slug = "gmail"+ tool.slug = "GMAIL_FETCH"++ with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):+ helper._download_file_value(+ {+ "name": "irrelevant.txt",+ "mimetype": "text/plain",+ "s3url": "https://example.com/f",+ },+ tool,+ )+++def test_file_helper_empty_string_outdir_is_treated_as_default(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ """``outdir=""`` falls through to the default directory (falsy, not just+ ``None``), so the default-ness flag must agree or the writability check+ silently stops firing for it."""+ unusable = tmp_path / "blocker-file"+ unusable.write_text("not a directory")+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable / "cache"))++ helper = FileHelper(client=Mock(), outdir="")+ assert helper._outdir_is_default is True+ assert helper._outdir == _files.get_output_file_directory()++ tool = Mock()+ tool.toolkit.slug = "gmail"+ tool.slug = "GMAIL_FETCH"++ with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):+ helper._download_file_value(+ {+ "name": "irrelevant.txt",+ "mimetype": "text/plain",+ "s3url": "https://example.com/f",+ },+ tool,+ )diff --git a/python/tests/test_path_join_guardrail.py b/python/tests/test_path_join_guardrail.pyindex fbb3cd9aa..4315adb8a 100644--- a/python/tests/test_path_join_guardrail.py+++ b/python/tests/test_path_join_guardrail.py@@ -58,13 +58,10 @@ def _reviewed_path(REVIEWED_JOINS: t.Dict[t.Tuple[str, str], t.Dict[str, t.Any]] = {("composio/core/models/_files.py",- (- "Path( os.environ.get( ENV_LOCAL_CACHE_DIRECTORY, Path.home() / "- "LOCAL_CACHE_DIRECTORY_NAME, # Fallback to user directory ) )"- ),+ "Path(configured)",): _reviewed_path(- "The cache root comes from local configuration or the user's home, not an "- "API response."+ "`configured` is read from the COMPOSIO_CACHE_DIR environment variable, "+ "which is local configuration, not an API response."),("composio/core/models/_files.py",