Skip to content

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

feat(py): address security concerns

jkomynob3f942cb19c9
diff --git a/python/composio/core/models/_files.py b/python/composio/core/models/_files.py
index b4bf0f328..52a076b2f 100644
--- a/python/composio/core/models/_files.py
+++ b/python/composio/core/models/_files.py
@@ -18,6 +18,7 @@ from composio.client.types import Tool
from composio.exceptions import (
ErrorDownloadingFile,
ErrorUploadingFile,
+ ResponseTooLargeError,
SDKFileNotFoundError,
)
from composio.utils import mimetypes
@@ -35,6 +36,20 @@ Long filenames (containing hashes, UUIDs, or encoded metadata) are replaced
with timestamped filenames to match TypeScript SDK behavior.
"""
+_MAX_RESPONSE_SIZE = 100 * 1024 * 1024 # 100 MB default limit
+"""
+Maximum response size in bytes when fetching files from URLs.
+Prevents memory exhaustion attacks from malicious URLs pointing to large files.
+"""
+
+_CONNECT_TIMEOUT = 5 # seconds
+_READ_TIMEOUT = 60 # seconds
+"""
+Separate connect and read timeouts for URL fetching.
+Connect timeout is short to fail fast on unreachable hosts.
+Read timeout is longer to allow for slower file transfers.
+"""
+
LOCAL_CACHE_DIRECTORY_NAME = ".composio"
"""
Local cache directory name for composio CLI
@@ -73,7 +88,20 @@ Local output file directory name for composio tools
"""
-def get_md5(file: Path):
+def get_md5(file: Path) -> str:
+ """Calculate MD5 hash of a file for integrity verification.
+
+ Note: MD5 is used here for file integrity checking and deduplication,
+ not for cryptographic security. The Composio API requires MD5 hashes
+ for file upload verification. For security-critical applications,
+ consider using SHA-256 for additional integrity checks.
+
+ Args:
+ file: Path to file to hash
+
+ Returns:
+ Hexadecimal MD5 hash string
+ """
obj = hashlib.md5()
with file.open("rb") as fp:
while True:
@@ -85,8 +113,18 @@ def get_md5(file: Path):
def upload(url: str, file: Path) -> bool:
+ """Upload file to presigned S3 URL.
+
+ Args:
+ url: Presigned S3 upload URL
+ file: Path to file to upload
+
+ Returns:
+ True if upload succeeded (HTTP 200), False otherwise
+ """
with file.open("rb") as data:
- return requests.put(url=url, data=data).status_code in (200, 403)
+ response = requests.put(url=url, data=data)
+ return response.status_code == 200
class _FileUploadResponse(_ComposioBaseModel):
@@ -168,19 +206,107 @@ def _truncate_filename(filename: str, max_length: int = _MAX_FILENAME_LENGTH) ->
return _generate_timestamped_filename(ext)
-def _fetch_file_from_url(url: str) -> t.Tuple[str, bytes, str]:
- """Fetch file content from a URL.
+def _sanitize_url_for_logging(url: str) -> str:
+ """Sanitize URL for safe logging (remove sensitive query params).
+
+ Args:
+ url: URL to sanitize
+
+ Returns:
+ URL with query parameters hidden
+ """
+ parsed = urlparse(url)
+ if parsed.query:
+ return f"{parsed.scheme}://{parsed.netloc}{parsed.path}?[REDACTED]"
+ return url
+
+
+def _fetch_file_from_url(
+ url: str,
+ max_size: int = _MAX_RESPONSE_SIZE,
+) -> t.Tuple[str, bytes, str]:
+ """Fetch file content from a URL with security protections.
+
+ Security features:
+ - Response size limiting (prevents memory exhaustion)
+ - Redirects disabled (prevents redirect-based attacks)
+ - Separate connect/read timeouts
+
+ Args:
+ url: URL to fetch file from
+ max_size: Maximum response size in bytes (default: 100MB)
Returns:
Tuple of (filename, content_bytes, mimetype)
+
+ Raises:
+ ResponseTooLargeError: If response exceeds max_size
+ ErrorUploadingFile: If fetch fails for other reasons
"""
- response = requests.get(url, timeout=30)
+ # Make request without following redirects
+ try:
+ response = requests.get(
+ url,
+ stream=True, # Enable streaming for size limiting
+ allow_redirects=False, # Disable redirects for security
+ timeout=(_CONNECT_TIMEOUT, _READ_TIMEOUT),
+ )
+ except requests.exceptions.Timeout:
+ raise ErrorUploadingFile(
+ f"Request timed out fetching URL: {_sanitize_url_for_logging(url)}"
+ )
+ except requests.exceptions.RequestException as e:
+ raise ErrorUploadingFile(
+ f"Failed to fetch file from URL: {_sanitize_url_for_logging(url)}. Error: {e}"
+ )
+
+ # Reject redirects - require direct URL to resource
+ if response.status_code in (301, 302, 303, 307, 308):
+ location = response.headers.get("Location", "unknown")
+ response.close()
+ raise ErrorUploadingFile(
+ f"URL returned redirect to {_sanitize_url_for_logging(location)}. "
+ f"Please provide a direct URL to the file."
+ )
+
+ # Check for successful response
if not response.ok:
+ response.close()
raise ErrorUploadingFile(
- f"Failed to fetch file from URL: {url}. Status: {response.status_code}"
+ f"Failed to fetch file from URL: {_sanitize_url_for_logging(url)}. "
+ f"Status: {response.status_code}"
+ )
+
+ # Check Content-Length header first (early abort for oversized files)
+ content_length = response.headers.get("Content-Length")
+ if content_length and int(content_length) > max_size:
+ response.close()
+ raise ResponseTooLargeError(
+ f"File size ({int(content_length)} bytes) exceeds maximum allowed "
+ f"size ({max_size} bytes)"
)
- content = response.content
+ # Stream response with size tracking
+ chunks: t.List[bytes] = []
+ total_bytes = 0
+ chunk_size = 8192 # 8 KB chunks
+
+ try:
+ for chunk in response.iter_content(chunk_size=chunk_size):
+ if chunk:
+ total_bytes += len(chunk)
+ if total_bytes > max_size:
+ response.close()
+ raise ResponseTooLargeError(
+ f"Response size exceeds maximum allowed size ({max_size} bytes)"
+ )
+ chunks.append(chunk)
+ finally:
+ response.close()
+
+ content = b"".join(chunks)
+
+ # Extract mimetype
mimetype = response.headers.get("content-type", "application/octet-stream")
# Handle mimetypes with charset or other parameters (e.g., "text/html; charset=utf-8")
mimetype = mimetype.split(";")[0].strip()
@@ -236,9 +362,10 @@ def _upload_bytes_to_s3(
headers={"Content-Type": mimetype},
)
- if upload_response.status_code not in (200, 403):
+ if upload_response.status_code != 200:
raise ErrorUploadingFile(
- f"Failed to upload to S3. Status: {upload_response.status_code}"
+ f"Failed to upload to S3. Status: {upload_response.status_code}. "
+ f"This may indicate an expired presigned URL or permission issue."
)
return s3meta.key
diff --git a/python/composio/exceptions.py b/python/composio/exceptions.py
index a944ca0df..f02131910 100644
--- a/python/composio/exceptions.py
+++ b/python/composio/exceptions.py
@@ -172,6 +172,12 @@ class ErrorDownloadingFile(FileError):
pass
+class ResponseTooLargeError(FileError):
+ """Raised when a response exceeds the maximum allowed size."""
+
+ pass
+
+
class TriggerError(ToolkitError):
pass
diff --git a/python/tests/test_files.py b/python/tests/test_files.py
index 48e4c76df..20b20c193 100644
--- a/python/tests/test_files.py
+++ b/python/tests/test_files.py
@@ -18,10 +18,11 @@ from composio.core.models._files import (
_truncate_filename,
_fetch_file_from_url,
_upload_bytes_to_s3,
+ _sanitize_url_for_logging,
_MAX_FILENAME_LENGTH,
)
from composio.core.models.base import allow_tracking
-from composio.exceptions import ErrorUploadingFile
+from composio.exceptions import ErrorUploadingFile, ResponseTooLargeError
@pytest.fixture(autouse=True)
@@ -1047,8 +1048,10 @@ class TestFetchFileFromUrl:
"""Test successful file fetch from URL."""
mock_response = MagicMock()
mock_response.ok = True
- mock_response.content = b"test file content"
+ mock_response.status_code = 200
mock_response.headers = {"content-type": "image/jpeg"}
+ mock_response.iter_content.return_value = [b"test file content"]
+ mock_response.close = MagicMock()
mock_get.return_value = mock_response
filename, content, mimetype = _fetch_file_from_url(
@@ -1058,15 +1061,22 @@ class TestFetchFileFromUrl:
assert content == b"test file content"
assert mimetype == "image/jpeg"
assert filename == "image.jpg"
- mock_get.assert_called_once_with("https://example.com/image.jpg", timeout=30)
+ mock_get.assert_called_once_with(
+ "https://example.com/image.jpg",
+ stream=True,
+ allow_redirects=False,
+ timeout=(5, 60),
+ )
@patch("composio.core.models._files.requests.get")
def test_fetch_file_from_url_with_charset_in_content_type(self, mock_get):
"""Test that charset is stripped from content-type."""
mock_response = MagicMock()
mock_response.ok = True
- mock_response.content = b"<html></html>"
+ mock_response.status_code = 200
mock_response.headers = {"content-type": "text/html; charset=utf-8"}
+ mock_response.iter_content.return_value = [b"<html></html>"]
+ mock_response.close = MagicMock()
mock_get.return_value = mock_response
filename, content, mimetype = _fetch_file_from_url(
@@ -1080,8 +1090,10 @@ class TestFetchFileFromUrl:
"""Test filename generation when URL has no filename."""
mock_response = MagicMock()
mock_response.ok = True
- mock_response.content = b"image data"
+ mock_response.status_code = 200
mock_response.headers = {"content-type": "image/png"}
+ mock_response.iter_content.return_value = [b"image data"]
+ mock_response.close = MagicMock()
mock_get.return_value = mock_response
filename, content, mimetype = _fetch_file_from_url("https://example.com/")
@@ -1094,8 +1106,10 @@ class TestFetchFileFromUrl:
"""Test filename generation when URL filename has no extension."""
mock_response = MagicMock()
mock_response.ok = True
- mock_response.content = b"pdf data"
+ mock_response.status_code = 200
mock_response.headers = {"content-type": "application/pdf"}
+ mock_response.iter_content.return_value = [b"pdf data"]
+ mock_response.close = MagicMock()
mock_get.return_value = mock_response
filename, content, mimetype = _fetch_file_from_url(
@@ -1111,6 +1125,7 @@ class TestFetchFileFromUrl:
mock_response = MagicMock()
mock_response.ok = False
mock_response.status_code = 404
+ mock_response.close = MagicMock()
mock_get.return_value = mock_response
with pytest.raises(ErrorUploadingFile) as exc_info:
@@ -1124,8 +1139,10 @@ class TestFetchFileFromUrl:
"""Test that percent-encoded characters in URL filenames are decoded."""
mock_response = MagicMock()
mock_response.ok = True
- mock_response.content = b"document content"
+ mock_response.status_code = 200
mock_response.headers = {"content-type": "application/pdf"}
+ mock_response.iter_content.return_value = [b"document content"]
+ mock_response.close = MagicMock()
mock_get.return_value = mock_response
# URL with percent-encoded spaces and special characters
@@ -1143,8 +1160,10 @@ class TestFetchFileFromUrl:
"""Test that percent-encoded unicode characters in URL filenames are decoded."""
mock_response = MagicMock()
mock_response.ok = True
- mock_response.content = b"image data"
+ mock_response.status_code = 200
mock_response.headers = {"content-type": "image/jpeg"}
+ mock_response.iter_content.return_value = [b"image data"]
+ mock_response.close = MagicMock()
mock_get.return_value = mock_response
# URL with percent-encoded unicode (e.g., Japanese characters)
@@ -1160,8 +1179,10 @@ class TestFetchFileFromUrl:
"""Test that plus signs in URL paths are preserved (not converted to spaces)."""
mock_response = MagicMock()
mock_response.ok = True
- mock_response.content = b"file content"
+ mock_response.status_code = 200
mock_response.headers = {"content-type": "text/plain"}
+ mock_response.iter_content.return_value = [b"file content"]
+ mock_response.close = MagicMock()
mock_get.return_value = mock_response
# Plus signs in path should remain as plus signs (unquote doesn't convert + to space)
@@ -1531,8 +1552,10 @@ class TestFetchFileFromUrlWithTruncation:
"""Long filenames from URLs should be truncated."""
mock_response = MagicMock()
mock_response.ok = True
- mock_response.content = b"test content"
+ mock_response.status_code = 200
mock_response.headers = {"content-type": "application/pdf"}
+ mock_response.iter_content.return_value = [b"test content"]
+ mock_response.close = MagicMock()
mock_get.return_value = mock_response
# Create a very long filename (hash-based, common in public buckets)
@@ -1552,8 +1575,10 @@ class TestFetchFileFromUrlWithTruncation:
"""Short filenames should be preserved unchanged."""
mock_response = MagicMock()
mock_response.ok = True
- mock_response.content = b"image data"
+ mock_response.status_code = 200
mock_response.headers = {"content-type": "image/jpeg"}
+ mock_response.iter_content.return_value = [b"image data"]
+ mock_response.close = MagicMock()
mock_get.return_value = mock_response
filename, content, mimetype = _fetch_file_from_url(
@@ -1567,8 +1592,10 @@ class TestFetchFileFromUrlWithTruncation:
"""Truncation should happen after extension is appended."""
mock_response = MagicMock()
mock_response.ok = True
- mock_response.content = b"pdf content"
+ mock_response.status_code = 200
mock_response.headers = {"content-type": "application/pdf"}
+ mock_response.iter_content.return_value = [b"pdf content"]
+ mock_response.close = MagicMock()
mock_get.return_value = mock_response
# URL without extension - extension will be added from mimetype
@@ -1586,8 +1613,10 @@ class TestFetchFileFromUrlWithTruncation:
"""Generated timestamped filenames (when URL has no filename) should be short enough."""
mock_response = MagicMock()
mock_response.ok = True
- mock_response.content = b"data"
+ mock_response.status_code = 200
mock_response.headers = {"content-type": "image/png"}
+ mock_response.iter_content.return_value = [b"data"]
+ mock_response.close = MagicMock()
mock_get.return_value = mock_response
# URL with no filename - will generate a timestamped one
@@ -1597,3 +1626,215 @@ class TestFetchFileFromUrlWithTruncation:
assert filename.startswith("file_")
assert filename.endswith(".png")
assert len(filename) < 50 # Timestamped names are short
+
+
+class TestResponseSizeLimit:
+ """Test response size limiting."""
+
+ @patch("composio.core.models._files.requests.get")
+ def test_rejects_oversized_content_length(self, mock_get):
+ """Files with Content-Length > max_size should be rejected early."""
+ mock_response = MagicMock()
+ mock_response.ok = True
+ mock_response.status_code = 200
+ mock_response.headers = {"Content-Length": "200000000"} # 200MB
+ mock_response.close = MagicMock()
+ mock_get.return_value = mock_response
+
+ with pytest.raises(ResponseTooLargeError):
+ _fetch_file_from_url(
+ "https://example.com/large.zip", max_size=100 * 1024 * 1024
+ )
+
+ @patch("composio.core.models._files.requests.get")
+ def test_rejects_oversized_during_streaming(self, mock_get):
+ """Files that exceed max_size during download should be rejected."""
+ mock_response = MagicMock()
+ mock_response.ok = True
+ mock_response.status_code = 200
+ mock_response.headers = {} # No Content-Length
+ # Return 20MB of data in chunks
+ mock_response.iter_content.return_value = [
+ b"x" * 1024 * 1024 for _ in range(20)
+ ]
+ mock_response.close = MagicMock()
+ mock_get.return_value = mock_response
+
+ with pytest.raises(ResponseTooLargeError):
+ _fetch_file_from_url(
+ "https://example.com/large.zip", max_size=10 * 1024 * 1024
+ )
+
+ @patch("composio.core.models._files.requests.get")
+ def test_accepts_file_within_limit(self, mock_get):
+ """Files within size limit should be accepted."""
+ mock_response = MagicMock()
+ mock_response.ok = True
+ mock_response.status_code = 200
+ mock_response.headers = {"content-type": "image/jpeg", "Content-Length": "1000"}
+ mock_response.iter_content.return_value = [b"x" * 1000]
+ mock_response.close = MagicMock()
+ mock_get.return_value = mock_response
+
+ filename, content, mimetype = _fetch_file_from_url(
+ "https://example.com/image.jpg", max_size=10 * 1024 * 1024
+ )
+
+ assert len(content) == 1000
+ assert mimetype == "image/jpeg"
+
+
+class TestRedirectHandling:
+ """Test redirect handling (redirects should be rejected)."""
+
+ @patch("composio.core.models._files.requests.get")
+ def test_rejects_redirect_302(self, mock_get):
+ """302 redirects should be rejected with clear error message."""
+ mock_response = MagicMock()
+ mock_response.status_code = 302
+ mock_response.headers = {"Location": "https://example.com/final.jpg"}
+ mock_response.close = MagicMock()
+ mock_get.return_value = mock_response
+
+ with pytest.raises(ErrorUploadingFile, match="redirect"):
+ _fetch_file_from_url("https://example.com/redirect")
+
+ @patch("composio.core.models._files.requests.get")
+ def test_rejects_redirect_301(self, mock_get):
+ """301 redirects should be rejected."""
+ mock_response = MagicMock()
+ mock_response.status_code = 301
+ mock_response.headers = {"Location": "https://example.com/"}
+ mock_response.close = MagicMock()
+ mock_get.return_value = mock_response
+
+ with pytest.raises(ErrorUploadingFile, match="redirect"):
+ _fetch_file_from_url("https://example.com/test")
+
+ @patch("composio.core.models._files.requests.get")
+ def test_rejects_redirect_307(self, mock_get):
+ """307 redirects should be rejected."""
+ mock_response = MagicMock()
+ mock_response.status_code = 307
+ mock_response.headers = {"Location": "https://example.com/"}
+ mock_response.close = MagicMock()
+ mock_get.return_value = mock_response
+
+ with pytest.raises(ErrorUploadingFile, match="redirect"):
+ _fetch_file_from_url("https://example.com/test")
+
+ @patch("composio.core.models._files.requests.get")
+ def test_rejects_redirect_308(self, mock_get):
+ """308 redirects should be rejected."""
+ mock_response = MagicMock()
+ mock_response.status_code = 308
+ mock_response.headers = {"Location": "https://example.com/"}
+ mock_response.close = MagicMock()
+ mock_get.return_value = mock_response
+
+ with pytest.raises(ErrorUploadingFile, match="redirect"):
+ _fetch_file_from_url("https://example.com/test")
+
+
+class TestS3UploadErrorHandling:
+ """Test S3 upload error handling."""
+
+ @patch("composio.core.models._files.requests.put")
+ def test_403_is_treated_as_error(self, mock_put):
+ """HTTP 403 should be treated as upload failure."""
+ mock_client = MagicMock()
+ mock_s3_response = MagicMock()
+ mock_s3_response.key = "s3-key"
+ mock_s3_response.new_presigned_url = "https://s3.example.com/upload"
+ mock_client.post.return_value = mock_s3_response
+
+ mock_put_response = MagicMock()
+ mock_put_response.status_code = 403
+ mock_put.return_value = mock_put_response
+
+ with pytest.raises(ErrorUploadingFile, match="403"):
+ _upload_bytes_to_s3(
+ client=mock_client,
+ filename="test.jpg",
+ content=b"data",
+ mimetype="image/jpeg",
+ tool="TEST",
+ toolkit="test",
+ )
+
+ @patch("composio.core.models._files.requests.put")
+ def test_200_is_success(self, mock_put):
+ """HTTP 200 should be treated as success."""
+ mock_client = MagicMock()
+ mock_s3_response = MagicMock()
+ mock_s3_response.key = "s3-key"
+ mock_s3_response.new_presigned_url = "https://s3.example.com/upload"
+ mock_client.post.return_value = mock_s3_response
+
+ mock_put_response = MagicMock()
+ mock_put_response.status_code = 200
+ mock_put.return_value = mock_put_response
+
+ result = _upload_bytes_to_s3(
+ client=mock_client,
+ filename="test.jpg",
+ content=b"data",
+ mimetype="image/jpeg",
+ tool="TEST",
+ toolkit="test",
+ )
+ assert result == "s3-key"
+
+ @patch("composio.core.models._files.requests.put")
+ def test_500_is_treated_as_error(self, mock_put):
+ """HTTP 500 should be treated as upload failure."""
+ mock_client = MagicMock()
+ mock_s3_response = MagicMock()
+ mock_s3_response.key = "s3-key"
+ mock_s3_response.new_presigned_url = "https://s3.example.com/upload"
+ mock_client.post.return_value = mock_s3_response
+
+ mock_put_response = MagicMock()
+ mock_put_response.status_code = 500
+ mock_put.return_value = mock_put_response
+
+ with pytest.raises(ErrorUploadingFile, match="500"):
+ _upload_bytes_to_s3(
+ client=mock_client,
+ filename="test.jpg",
+ content=b"data",
+ mimetype="image/jpeg",
+ tool="TEST",
+ toolkit="test",
+ )
+
+
+class TestUrlSanitization:
+ """Test URL sanitization for logging."""
+
+ def test_sanitizes_query_params(self):
+ """Query parameters should be redacted in logs."""
+ url = "https://example.com/file?token=secret123&key=abc"
+ sanitized = _sanitize_url_for_logging(url)
+ assert "secret123" not in sanitized
+ assert "abc" not in sanitized
+ assert "[REDACTED]" in sanitized
+
+ def test_preserves_url_without_query(self):
+ """URLs without query params should be unchanged."""
+ url = "https://example.com/path/to/file.jpg"
+ sanitized = _sanitize_url_for_logging(url)
+ assert sanitized == url
+
+ def test_preserves_path(self):
+ """Path should be preserved when redacting query params."""
+ url = "https://example.com/path/to/file.jpg?token=secret"
+ sanitized = _sanitize_url_for_logging(url)
+ assert "/path/to/file.jpg" in sanitized
+ assert "example.com" in sanitized
+
+ def test_handles_empty_query(self):
+ """URLs with empty query string should not have [REDACTED]."""
+ url = "https://example.com/file.jpg"
+ sanitized = _sanitize_url_for_logging(url)
+ assert "[REDACTED]" not in sanitized