Skip to content

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

fix(python): create the cache directory on first use instead of at import time (#4162)

Alberto Schiabel03429c74044b
diff --git a/python/composio/core/models/_files.py b/python/composio/core/models/_files.py
index dd3c5e84e..cd3dcbd5a 100644
--- a/python/composio/core/models/_files.py
+++ b/python/composio/core/models/_files.py
@@ -82,35 +82,85 @@ ENV_LOCAL_CACHE_DIRECTORY = "COMPOSIO_CACHE_DIR"
Environment to set the composio caching directory.
"""
-LOCAL_CACHE_DIRECTORY = Path(
- os.environ.get(
- ENV_LOCAL_CACHE_DIRECTORY,
- Path.home() / LOCAL_CACHE_DIRECTORY_NAME, # Fallback to user directory
- )
-)
+LOCAL_OUTPUT_FILE_DIRECTORY_NAME = "files"
"""
-Path to local caching directory.
+Name of the cache sub-directory into which files downloaded during tool
+execution are written. Previously ``outputs``; now ``files`` for parity with
+the TypeScript SDK.
"""
-try:
- LOCAL_CACHE_DIRECTORY.mkdir(parents=True, exist_ok=True)
- if not os.access(LOCAL_CACHE_DIRECTORY, os.W_OK):
- raise OSError
-except OSError as e:
- raise RuntimeError(
- f"Cache directory {LOCAL_CACHE_DIRECTORY} is not writable please "
- f"provide a path that is writable using {ENV_LOCAL_CACHE_DIRECTORY} "
- "environment variable."
- ) from e
+def get_cache_directory() -> Path:
+ """Resolve the local caching directory without touching the filesystem.
-LOCAL_OUTPUT_FILE_DIRECTORY = LOCAL_CACHE_DIRECTORY / "files"
-"""
-Default local directory into which files downloaded during tool execution are
-written. Previously ``<cache>/outputs``; now ``<cache>/files`` for parity with
-the TypeScript SDK. Override by passing ``file_download_dir=...`` to Composio,
-or by setting ``outdir`` on ``FileHelper`` directly.
-"""
+ ``COMPOSIO_CACHE_DIR`` is read on every call, so it can be set after
+ ``composio`` has already been imported. ``Path.home()`` is only consulted
+ when the variable is unset: it can raise ``RuntimeError`` when there is no
+ resolvable home directory, which is exactly the situation
+ ``COMPOSIO_CACHE_DIR`` exists to work around, so it must not be evaluated
+ eagerly as a fallback argument.
+ """
+ configured = os.environ.get(ENV_LOCAL_CACHE_DIRECTORY)
+ if configured:
+ return Path(configured)
+
+ try:
+ home = Path.home()
+ except RuntimeError as e:
+ raise RuntimeError(
+ "Could not determine a home directory to store the Composio cache "
+ f"in. Provide a writable path using the {ENV_LOCAL_CACHE_DIRECTORY} "
+ "environment variable."
+ ) from e
+ return home / LOCAL_CACHE_DIRECTORY_NAME
+
+
+def get_output_file_directory() -> Path:
+ """Default local directory into which files downloaded during tool
+ execution are written. Override by passing ``file_download_dir=...`` to
+ Composio, or by setting ``outdir`` on ``FileHelper`` directly.
+ """
+ return get_cache_directory() / LOCAL_OUTPUT_FILE_DIRECTORY_NAME
+
+
+def ensure_cache_directory() -> Path:
+ """Create the cache directory on first use and check that it is writable.
+
+ This used to run at module import time, so a bare ``import composio``
+ raised ``RuntimeError`` on any read-only filesystem -- AWS Lambda,
+ distroless containers, ``ProtectHome=true`` systemd units -- even for
+ programs that never touched a file. Deferring it to first use keeps the
+ same check, and the same error message, for the callers that actually
+ need the directory.
+ """
+ directory = get_cache_directory()
+ try:
+ directory.mkdir(parents=True, exist_ok=True)
+ if not os.access(directory, os.W_OK):
+ raise OSError
+ except OSError as e:
+ raise RuntimeError(
+ f"Cache directory {directory} is not writable please "
+ f"provide a path that is writable using {ENV_LOCAL_CACHE_DIRECTORY} "
+ "environment variable."
+ ) from e
+ return directory
+
+
+def __getattr__(name: str) -> Path:
+ """Keep the historical module-level path constants working, but lazily.
+
+ ``LOCAL_CACHE_DIRECTORY`` and ``LOCAL_OUTPUT_FILE_DIRECTORY`` used to be
+ computed at import time. They are now resolved on attribute access
+ instead (PEP 562), so importing this module no longer touches the
+ filesystem or depends on the environment, and both constants observe a
+ ``COMPOSIO_CACHE_DIR`` that was set after import.
+ """
+ if name == "LOCAL_CACHE_DIRECTORY":
+ return get_cache_directory()
+ if name == "LOCAL_OUTPUT_FILE_DIRECTORY":
+ return get_output_file_directory()
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
def get_md5(file: Path) -> str:
@@ -670,7 +720,11 @@ class FileHelper(WithLogger):
"""
super().__init__()
self._client = client
- self._outdir = Path(outdir) if outdir else LOCAL_OUTPUT_FILE_DIRECTORY
+ # Falsy, not just `is None`: an empty string falls through to the
+ # same default directory below, and must be treated as such here too
+ # or `ensure_cache_directory()` silently stops firing for it.
+ self._outdir_is_default = not outdir
+ self._outdir = Path(outdir) if outdir else get_output_file_directory()
self._sensitive_file_upload_protection = sensitive_file_upload_protection
self._file_upload_path_deny_segments = file_upload_path_deny_segments
self._file_upload_allowlist: t.Optional[t.Sequence[Path]] = (
@@ -1103,6 +1157,11 @@ class FileHelper(WithLogger):
def _download_file_value(self, value: t.Any, tool: Tool) -> t.Any:
if isinstance(value, dict) and "s3url" in value:
+ if self._outdir_is_default:
+ # First point at which the cache directory is genuinely
+ # needed. Raises the same RuntimeError that used to be raised
+ # at import time, pointing at COMPOSIO_CACHE_DIR.
+ ensure_cache_directory()
# `tool.toolkit.slug` and `tool.slug` come from the API response and
# are untrusted, so joining them directly would let the response pick
# the download directory. `secure_join` validates each component and
diff --git a/python/tests/test_files.py b/python/tests/test_files.py
index c4a7bd60c..ac6dda819 100644
--- a/python/tests/test_files.py
+++ b/python/tests/test_files.py
@@ -52,8 +52,15 @@ def mock_client():
@pytest.fixture
-def file_helper(mock_client):
- """Create a FileHelper instance with a mock client."""
+def file_helper(mock_client, monkeypatch, tmp_path):
+ """Create a FileHelper instance with a mock client.
+
+ Sandboxes ``COMPOSIO_CACHE_DIR`` so the default-outdir download path
+ (``ensure_cache_directory()``) never touches the real home directory,
+ even though ``FileDownloadable.download`` itself is mocked in most of
+ these tests.
+ """
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / ".composio"))
return FileHelper(client=mock_client)
diff --git a/python/tests/test_lazy_cache_directory.py b/python/tests/test_lazy_cache_directory.py
new file mode 100644
index 000000000..eadc5eda2
--- /dev/null
+++ b/python/tests/test_lazy_cache_directory.py
@@ -0,0 +1,248 @@
+"""Regression tests for lazy creation of the Composio cache directory.
+
+``composio/core/models/_files.py`` used to create ``~/.composio`` and assert
+that it was writable at *module import time*. Because ``composio/__init__.py``
+imports ``tools.py``, which imports ``_files.py`` unconditionally, a bare
+``import composio`` raised ``RuntimeError`` on any read-only filesystem --
+AWS Lambda, distroless containers, ``ProtectHome=true`` systemd units -- even
+for programs that never touched a file (#4150).
+
+These tests pin the fix: importing the package performs no filesystem work,
+``COMPOSIO_CACHE_DIR`` is honoured when set after import, and the writability
+check still raises the same error when the directory is genuinely needed.
+"""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+from unittest.mock import Mock
+
+import pytest
+
+from composio.core.models import _files
+from composio.core.models._files import FileHelper
+from composio.exceptions import BlockedInternalUrlError
+
+
+def unusable_cache_dir(tmp_path: Path) -> Path:
+ """A cache path that cannot be created, without relying on permissions.
+
+ ``chmod``-based read-only directories are a no-op when the test suite
+ runs as root (``os.access(..., W_OK)`` returns ``True`` for uid 0), which
+ is common in containerized CI. Nesting the cache directory *underneath a
+ regular file* makes ``mkdir`` fail with ``NotADirectoryError`` for every
+ user.
+ """
+ blocker = tmp_path / "blocker"
+ blocker.write_text("not a directory")
+ return blocker / "cache"
+
+
+def test_importing_composio_does_not_create_the_cache_directory(
+ unusable_cache_dir: Path,
+) -> None:
+ """The regression test: ``import composio`` must not touch the filesystem.
+
+ Run in a subprocess so the import genuinely happens under the patched
+ environment rather than hitting the already-imported module in this
+ process.
+ """
+ env = dict(os.environ)
+ env["COMPOSIO_CACHE_DIR"] = str(unusable_cache_dir)
+
+ result = subprocess.run(
+ [sys.executable, "-c", "import composio"],
+ env=env,
+ capture_output=True,
+ text=True,
+ timeout=120,
+ )
+
+ assert result.returncode == 0, (
+ "importing composio with an uncreatable cache directory must succeed, "
+ f"but it failed with:\n{result.stderr}"
+ )
+ assert not unusable_cache_dir.exists()
+
+
+def test_cache_directory_is_created_only_on_demand(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ cache_dir = tmp_path / "not-yet-there"
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(cache_dir))
+
+ # Merely resolving the path must not create it.
+ assert _files.get_cache_directory() == cache_dir
+ assert not cache_dir.exists()
+
+ assert _files.ensure_cache_directory() == cache_dir
+ assert cache_dir.is_dir()
+
+
+def test_cache_dir_env_var_is_read_after_import(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ """``COMPOSIO_CACHE_DIR`` used to be frozen at import time."""
+ first = tmp_path / "first"
+ second = tmp_path / "second"
+
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(first))
+ assert _files.get_cache_directory() == first
+
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(second))
+ assert _files.get_cache_directory() == second
+
+
+def test_home_is_not_resolved_when_the_env_var_is_set(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ """The escape hatch has to work when ``Path.home()`` itself fails.
+
+ The old code passed ``Path.home() / ...`` as the *default argument* to
+ ``os.environ.get``, which Python evaluates eagerly, so setting
+ ``COMPOSIO_CACHE_DIR`` did not save a user whose home directory could not
+ be resolved.
+ """
+
+ def _explode() -> Path:
+ raise RuntimeError("Could not determine home directory")
+
+ monkeypatch.setattr(Path, "home", staticmethod(_explode))
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "cache"))
+
+ assert _files.get_cache_directory() == tmp_path / "cache"
+
+
+def test_unresolvable_home_raises_a_helpful_error(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ def _explode() -> Path:
+ raise RuntimeError("Could not determine home directory")
+
+ monkeypatch.setattr(Path, "home", staticmethod(_explode))
+ monkeypatch.delenv("COMPOSIO_CACHE_DIR", raising=False)
+
+ with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):
+ _files.get_cache_directory()
+
+
+def test_output_file_directory_hangs_off_the_cache_directory(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "cache"))
+ assert _files.get_output_file_directory() == tmp_path / "cache" / "files"
+
+
+def test_ensure_cache_directory_still_reports_unwritable_paths(
+ monkeypatch: pytest.MonkeyPatch, unusable_cache_dir: Path
+) -> None:
+ """The writability check is deferred, not removed."""
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable_cache_dir))
+
+ with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):
+ _files.ensure_cache_directory()
+
+
+def test_legacy_module_constants_still_resolve(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ """``LOCAL_CACHE_DIRECTORY`` was public-ish; keep it importable (PEP 562)."""
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "cache"))
+
+ assert _files.LOCAL_CACHE_DIRECTORY == tmp_path / "cache"
+ assert _files.LOCAL_OUTPUT_FILE_DIRECTORY == tmp_path / "cache" / "files"
+
+ # And unlike the old module-level constants, they track the environment.
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(tmp_path / "other"))
+ assert _files.LOCAL_CACHE_DIRECTORY == tmp_path / "other"
+
+
+def test_unknown_module_attribute_still_raises_attribute_error() -> None:
+ """The module ``__getattr__`` must not swallow genuine typos."""
+ with pytest.raises(AttributeError, match="no attribute"):
+ _files.LOCAL_CACHE_DIRECTROY # noqa: B018 (deliberate typo)
+
+
+def test_file_helper_with_explicit_outdir_never_touches_the_default_cache(
+ monkeypatch: pytest.MonkeyPatch, unusable_cache_dir: Path, tmp_path: Path
+) -> None:
+ """A caller-supplied ``outdir`` must not force the default cache directory
+ into existence, and downloads into it must not trigger the writability
+ check for the (unused) default cache directory."""
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable_cache_dir))
+
+ outdir = tmp_path / "custom-outdir"
+ helper = FileHelper(client=Mock(), outdir=str(outdir))
+ assert helper._outdir_is_default is False
+
+ tool = Mock()
+ tool.toolkit.slug = "gmail"
+ tool.slug = "GMAIL_FETCH"
+
+ # The download itself fails fast on the blank s3url, but the point of
+ # this test is what happened *before* that: the default cache directory
+ # must never have been created.
+ with pytest.raises(BlockedInternalUrlError):
+ helper._download_file_value(
+ {"name": "irrelevant.txt", "mimetype": "text/plain", "s3url": ""},
+ tool,
+ )
+ assert not unusable_cache_dir.exists()
+
+
+def test_file_helper_default_outdir_ensures_cache_directory_on_download(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ """The default-outdir path is exactly where the deferred writability
+ check must still fire, with the original error message."""
+ unusable = tmp_path / "blocker-file"
+ unusable.write_text("not a directory")
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable / "cache"))
+
+ helper = FileHelper(client=Mock())
+ assert helper._outdir_is_default is True
+
+ tool = Mock()
+ tool.toolkit.slug = "gmail"
+ tool.slug = "GMAIL_FETCH"
+
+ with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):
+ helper._download_file_value(
+ {
+ "name": "irrelevant.txt",
+ "mimetype": "text/plain",
+ "s3url": "https://example.com/f",
+ },
+ tool,
+ )
+
+
+def test_file_helper_empty_string_outdir_is_treated_as_default(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ """``outdir=""`` falls through to the default directory (falsy, not just
+ ``None``), so the default-ness flag must agree or the writability check
+ silently stops firing for it."""
+ unusable = tmp_path / "blocker-file"
+ unusable.write_text("not a directory")
+ monkeypatch.setenv("COMPOSIO_CACHE_DIR", str(unusable / "cache"))
+
+ helper = FileHelper(client=Mock(), outdir="")
+ assert helper._outdir_is_default is True
+ assert helper._outdir == _files.get_output_file_directory()
+
+ tool = Mock()
+ tool.toolkit.slug = "gmail"
+ tool.slug = "GMAIL_FETCH"
+
+ with pytest.raises(RuntimeError, match="COMPOSIO_CACHE_DIR"):
+ helper._download_file_value(
+ {
+ "name": "irrelevant.txt",
+ "mimetype": "text/plain",
+ "s3url": "https://example.com/f",
+ },
+ tool,
+ )
diff --git a/python/tests/test_path_join_guardrail.py b/python/tests/test_path_join_guardrail.py
index fbb3cd9aa..4315adb8a 100644
--- a/python/tests/test_path_join_guardrail.py
+++ b/python/tests/test_path_join_guardrail.py
@@ -58,13 +58,10 @@ def _reviewed_path(
REVIEWED_JOINS: t.Dict[t.Tuple[str, str], t.Dict[str, t.Any]] = {
(
"composio/core/models/_files.py",
- (
- "Path( os.environ.get( ENV_LOCAL_CACHE_DIRECTORY, Path.home() / "
- "LOCAL_CACHE_DIRECTORY_NAME, # Fallback to user directory ) )"
- ),
+ "Path(configured)",
): _reviewed_path(
- "The cache root comes from local configuration or the user's home, not an "
- "API response."
+ "`configured` is read from the COMPOSIO_CACHE_DIR environment variable, "
+ "which is local configuration, not an API response."
),
(
"composio/core/models/_files.py",