Skip to content

Composio powers 1000+ toolkits, tool search, context management, authentication, and a sandboxed workbench to help you build AI agents that turn intent into action.

feat(py): enable public URLs

jkomynof912596ae75a
diff --git a/python/composio/core/models/_files.py b/python/composio/core/models/_files.py
index 27377c4ef..146a19e7e 100644
--- a/python/composio/core/models/_files.py
+++ b/python/composio/core/models/_files.py
@@ -4,6 +4,9 @@ import hashlib
import os
import typing as t
from pathlib import Path
+from urllib.parse import urlparse
+import uuid
+import datetime
import requests
import typing_extensions as te
@@ -87,6 +90,122 @@ class _FileUploadResponse(_ComposioBaseModel):
new_presigned_url: str
+def _is_url(value: str) -> bool:
+ """Check if a string is a valid HTTP/HTTPS URL."""
+ try:
+ parsed = urlparse(value)
+ return parsed.scheme in ("http", "https") and bool(parsed.netloc)
+ except Exception:
+ return False
+
+
+def _get_extension_from_mimetype(mimetype: str) -> str:
+ """Get file extension from mimetype."""
+ mime_to_ext = {
+ "image/jpeg": ".jpg",
+ "image/png": ".png",
+ "image/gif": ".gif",
+ "image/webp": ".webp",
+ "image/svg+xml": ".svg",
+ "image/bmp": ".bmp",
+ "image/tiff": ".tiff",
+ "application/pdf": ".pdf",
+ "application/json": ".json",
+ "application/xml": ".xml",
+ "text/plain": ".txt",
+ "text/html": ".html",
+ "text/css": ".css",
+ "text/javascript": ".js",
+ "application/zip": ".zip",
+ "application/gzip": ".gz",
+ "audio/mpeg": ".mp3",
+ "audio/wav": ".wav",
+ "video/mp4": ".mp4",
+ "video/webm": ".webm",
+ }
+ return mime_to_ext.get(mimetype, "")
+
+
+def _generate_timestamped_filename(extension: str) -> str:
+ """Generate a unique filename with timestamp."""
+ timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+ unique_id = uuid.uuid4().hex[:8]
+ return f"file_{timestamp}_{unique_id}{extension}"
+
+
+def _fetch_file_from_url(url: str) -> t.Tuple[str, bytes, str]:
+ """Fetch file content from a URL.
+
+ Returns:
+ Tuple of (filename, content_bytes, mimetype)
+ """
+ response = requests.get(url, timeout=30)
+ if not response.ok:
+ raise ErrorUploadingFile(
+ f"Failed to fetch file from URL: {url}. Status: {response.status_code}"
+ )
+
+ content = response.content
+ mimetype = response.headers.get("content-type", "application/octet-stream")
+ # Handle mimetypes with charset or other parameters (e.g., "text/html; charset=utf-8")
+ mimetype = mimetype.split(";")[0].strip()
+
+ # Extract filename from URL
+ parsed_url = urlparse(url)
+ pathname = parsed_url.path
+ filename = os.path.basename(pathname) if pathname else ""
+
+ # If no filename from URL or no extension, generate one
+ if not filename or filename == "/":
+ extension = _get_extension_from_mimetype(mimetype)
+ filename = _generate_timestamped_filename(extension)
+ else:
+ # If filename has no extension, try to add one from mimetype
+ if "." not in filename:
+ extension = _get_extension_from_mimetype(mimetype)
+ filename = _generate_timestamped_filename(extension)
+
+ return filename, content, mimetype
+
+
+def _upload_bytes_to_s3(
+ client: HttpClient,
+ filename: str,
+ content: bytes,
+ mimetype: str,
+ tool: str,
+ toolkit: str,
+) -> str:
+ """Upload bytes content to S3 and return the S3 key."""
+ md5_hash = hashlib.md5(content).hexdigest()
+
+ s3meta = client.post(
+ path=_FILE_UPLOAD,
+ body={
+ "md5": md5_hash,
+ "filename": filename,
+ "mimetype": mimetype,
+ "tool_slug": tool,
+ "toolkit_slug": toolkit,
+ },
+ cast_to=_FileUploadResponse,
+ )
+
+ # Upload the content directly to S3
+ upload_response = requests.put(
+ url=s3meta.new_presigned_url,
+ data=content,
+ headers={"Content-Type": mimetype},
+ )
+
+ if upload_response.status_code not in (200, 403):
+ raise ErrorUploadingFile(
+ f"Failed to upload to S3. Status: {upload_response.status_code}"
+ )
+
+ return s3meta.key
+
+
class FileUploadable(BaseModel):
model_config = ConfigDict(json_schema_extra={"file_uploadable": True})
@@ -94,6 +213,37 @@ class FileUploadable(BaseModel):
mimetype: str
s3key: str
+ @classmethod
+ def from_url(
+ cls,
+ client: HttpClient,
+ url: str,
+ tool: str,
+ toolkit: str,
+ ) -> te.Self:
+ """Create a FileUploadable from a public URL.
+
+ Fetches the file content from the URL and uploads it to S3.
+
+ :param client: The HTTP client for API calls
+ :param url: The public URL to fetch the file from
+ :param tool: The tool slug
+ :param toolkit: The toolkit slug
+ :return: FileUploadable instance with S3 key
+ """
+ filename, content, mimetype = _fetch_file_from_url(url)
+
+ s3key = _upload_bytes_to_s3(
+ client=client,
+ filename=filename,
+ content=content,
+ mimetype=mimetype,
+ tool=tool,
+ toolkit=toolkit,
+ )
+
+ return cls(name=filename, mimetype=mimetype, s3key=s3key)
+
@classmethod
def from_path(
cls,
@@ -102,6 +252,23 @@ class FileUploadable(BaseModel):
tool: str,
toolkit: str,
) -> te.Self:
+ """Create a FileUploadable from a local file path or public URL.
+
+ If the file parameter is a URL (starts with http:// or https://),
+ it will fetch the file content from the URL and upload it to S3.
+ Otherwise, it treats it as a local file path.
+
+ :param client: The HTTP client for API calls
+ :param file: Local file path or public URL
+ :param tool: The tool slug
+ :param toolkit: The toolkit slug
+ :return: FileUploadable instance with S3 key
+ """
+ # Check if it's a URL
+ if isinstance(file, str) and _is_url(file):
+ return cls.from_url(client=client, url=file, tool=tool, toolkit=toolkit)
+
+ # Handle as local file path
file = Path(file)
if not file.exists():
raise SDKFileNotFoundError(
diff --git a/python/tests/test_files.py b/python/tests/test_files.py
index b37b1daca..9cf5eda24 100644
--- a/python/tests/test_files.py
+++ b/python/tests/test_files.py
@@ -4,13 +4,22 @@ These tests ensure that the FileHelper class correctly handles JSON schemas
that use anyOf, oneOf, allOf, or $ref instead of direct 'type' properties.
"""
-from unittest.mock import Mock
+from unittest.mock import Mock, patch, MagicMock
import pytest
from composio.client.types import Tool, tool_list_response
-from composio.core.models._files import FileHelper
+from composio.core.models._files import (
+ FileHelper,
+ FileUploadable,
+ _is_url,
+ _get_extension_from_mimetype,
+ _generate_timestamped_filename,
+ _fetch_file_from_url,
+ _upload_bytes_to_s3,
+)
from composio.core.models.base import allow_tracking
+from composio.exceptions import ErrorUploadingFile
@pytest.fixture(autouse=True)
@@ -954,3 +963,377 @@ class TestFileUploadWithMixedSchemas:
# Both values should be preserved unchanged
assert result["metadata"] == "test data"
assert result["count"] == 42
+
+
+class TestUrlHelperFunctions:
+ """Test cases for URL-related helper functions."""
+
+ def test_is_url_with_http(self):
+ """Test _is_url correctly identifies HTTP URLs."""
+ assert _is_url("http://example.com/file.jpg") is True
+ assert _is_url("http://localhost:8080/api/file") is True
+
+ def test_is_url_with_https(self):
+ """Test _is_url correctly identifies HTTPS URLs."""
+ assert _is_url("https://example.com/file.jpg") is True
+ assert _is_url("https://images.pexels.com/photos/123.jpg") is True
+
+ def test_is_url_with_local_paths(self):
+ """Test _is_url correctly rejects local file paths."""
+ assert _is_url("/path/to/file.jpg") is False
+ assert _is_url("./relative/path.txt") is False
+ assert _is_url("file.txt") is False
+ assert _is_url("C:\\Windows\\file.txt") is False
+
+ def test_is_url_with_other_schemes(self):
+ """Test _is_url rejects non-HTTP schemes."""
+ assert _is_url("ftp://ftp.example.com/file.txt") is False
+ assert _is_url("file:///local/file.txt") is False
+ assert _is_url("mailto:[email protected]") is False
+
+ def test_is_url_with_invalid_inputs(self):
+ """Test _is_url handles invalid inputs gracefully."""
+ assert _is_url("") is False
+ assert _is_url("not a url") is False
+ assert _is_url("http://") is False
+
+ def test_get_extension_from_mimetype_common_types(self):
+ """Test _get_extension_from_mimetype for common types."""
+ assert _get_extension_from_mimetype("image/jpeg") == ".jpg"
+ assert _get_extension_from_mimetype("image/png") == ".png"
+ assert _get_extension_from_mimetype("application/pdf") == ".pdf"
+ assert _get_extension_from_mimetype("text/plain") == ".txt"
+ assert _get_extension_from_mimetype("application/json") == ".json"
+
+ def test_get_extension_from_mimetype_unknown_type(self):
+ """Test _get_extension_from_mimetype returns empty for unknown types."""
+ assert _get_extension_from_mimetype("application/unknown") == ""
+ assert _get_extension_from_mimetype("custom/type") == ""
+
+ def test_generate_timestamped_filename(self):
+ """Test _generate_timestamped_filename generates valid filenames."""
+ filename = _generate_timestamped_filename(".jpg")
+ assert filename.startswith("file_")
+ assert filename.endswith(".jpg")
+ assert len(filename) > 15 # Should have timestamp and unique ID
+
+ def test_generate_timestamped_filename_no_extension(self):
+ """Test _generate_timestamped_filename works without extension."""
+ filename = _generate_timestamped_filename("")
+ assert filename.startswith("file_")
+ assert not filename.endswith(".")
+
+
+class TestFetchFileFromUrl:
+ """Test cases for _fetch_file_from_url function."""
+
+ @patch("composio.core.models._files.requests.get")
+ def test_fetch_file_from_url_success(self, mock_get):
+ """Test successful file fetch from URL."""
+ mock_response = MagicMock()
+ mock_response.ok = True
+ mock_response.content = b"test file content"
+ mock_response.headers = {"content-type": "image/jpeg"}
+ mock_get.return_value = mock_response
+
+ filename, content, mimetype = _fetch_file_from_url(
+ "https://example.com/image.jpg"
+ )
+
+ assert content == b"test file content"
+ assert mimetype == "image/jpeg"
+ assert filename == "image.jpg"
+ mock_get.assert_called_once_with("https://example.com/image.jpg", timeout=30)
+
+ @patch("composio.core.models._files.requests.get")
+ def test_fetch_file_from_url_with_charset_in_content_type(self, mock_get):
+ """Test that charset is stripped from content-type."""
+ mock_response = MagicMock()
+ mock_response.ok = True
+ mock_response.content = b"<html></html>"
+ mock_response.headers = {"content-type": "text/html; charset=utf-8"}
+ mock_get.return_value = mock_response
+
+ filename, content, mimetype = _fetch_file_from_url(
+ "https://example.com/page.html"
+ )
+
+ assert mimetype == "text/html"
+
+ @patch("composio.core.models._files.requests.get")
+ def test_fetch_file_from_url_generates_filename_when_missing(self, mock_get):
+ """Test filename generation when URL has no filename."""
+ mock_response = MagicMock()
+ mock_response.ok = True
+ mock_response.content = b"image data"
+ mock_response.headers = {"content-type": "image/png"}
+ mock_get.return_value = mock_response
+
+ filename, content, mimetype = _fetch_file_from_url("https://example.com/")
+
+ assert filename.startswith("file_")
+ assert filename.endswith(".png")
+
+ @patch("composio.core.models._files.requests.get")
+ def test_fetch_file_from_url_generates_filename_when_no_extension(self, mock_get):
+ """Test filename generation when URL filename has no extension."""
+ mock_response = MagicMock()
+ mock_response.ok = True
+ mock_response.content = b"pdf data"
+ mock_response.headers = {"content-type": "application/pdf"}
+ mock_get.return_value = mock_response
+
+ filename, content, mimetype = _fetch_file_from_url(
+ "https://example.com/document"
+ )
+
+ assert filename.startswith("file_")
+ assert filename.endswith(".pdf")
+
+ @patch("composio.core.models._files.requests.get")
+ def test_fetch_file_from_url_failure(self, mock_get):
+ """Test error handling when URL fetch fails."""
+ mock_response = MagicMock()
+ mock_response.ok = False
+ mock_response.status_code = 404
+ mock_get.return_value = mock_response
+
+ with pytest.raises(ErrorUploadingFile) as exc_info:
+ _fetch_file_from_url("https://example.com/notfound.jpg")
+
+ assert "Failed to fetch file from URL" in str(exc_info.value)
+ assert "404" in str(exc_info.value)
+
+
+class TestUploadBytesToS3:
+ """Test cases for _upload_bytes_to_s3 function."""
+
+ @patch("composio.core.models._files.requests.put")
+ def test_upload_bytes_to_s3_success(self, mock_put):
+ """Test successful upload to S3."""
+ mock_client = MagicMock()
+ mock_s3_response = MagicMock()
+ mock_s3_response.key = "s3-key-123"
+ mock_s3_response.new_presigned_url = "https://s3.example.com/upload"
+ mock_client.post.return_value = mock_s3_response
+
+ mock_put_response = MagicMock()
+ mock_put_response.status_code = 200
+ mock_put.return_value = mock_put_response
+
+ result = _upload_bytes_to_s3(
+ client=mock_client,
+ filename="test.jpg",
+ content=b"file content",
+ mimetype="image/jpeg",
+ tool="TEST_TOOL",
+ toolkit="test_toolkit",
+ )
+
+ assert result == "s3-key-123"
+ mock_client.post.assert_called_once()
+ mock_put.assert_called_once()
+
+ @patch("composio.core.models._files.requests.put")
+ def test_upload_bytes_to_s3_failure(self, mock_put):
+ """Test error handling when S3 upload fails."""
+ mock_client = MagicMock()
+ mock_s3_response = MagicMock()
+ mock_s3_response.key = "s3-key-123"
+ mock_s3_response.new_presigned_url = "https://s3.example.com/upload"
+ mock_client.post.return_value = mock_s3_response
+
+ mock_put_response = MagicMock()
+ mock_put_response.status_code = 500
+ mock_put.return_value = mock_put_response
+
+ with pytest.raises(ErrorUploadingFile) as exc_info:
+ _upload_bytes_to_s3(
+ client=mock_client,
+ filename="test.jpg",
+ content=b"file content",
+ mimetype="image/jpeg",
+ tool="TEST_TOOL",
+ toolkit="test_toolkit",
+ )
+
+ assert "Failed to upload to S3" in str(exc_info.value)
+
+
+class TestFileUploadableFromUrl:
+ """Test cases for FileUploadable.from_url and from_path with URLs."""
+
+ @patch("composio.core.models._files._upload_bytes_to_s3")
+ @patch("composio.core.models._files._fetch_file_from_url")
+ def test_from_url_success(self, mock_fetch, mock_upload):
+ """Test successful FileUploadable creation from URL."""
+ mock_fetch.return_value = ("image.jpg", b"image data", "image/jpeg")
+ mock_upload.return_value = "s3-key-abc"
+ mock_client = MagicMock()
+
+ result = FileUploadable.from_url(
+ client=mock_client,
+ url="https://example.com/image.jpg",
+ tool="TEST_TOOL",
+ toolkit="test_toolkit",
+ )
+
+ assert result.name == "image.jpg"
+ assert result.mimetype == "image/jpeg"
+ assert result.s3key == "s3-key-abc"
+ mock_fetch.assert_called_once_with("https://example.com/image.jpg")
+ mock_upload.assert_called_once()
+
+ @patch("composio.core.models._files._upload_bytes_to_s3")
+ @patch("composio.core.models._files._fetch_file_from_url")
+ def test_from_path_detects_url(self, mock_fetch, mock_upload):
+ """Test that from_path correctly detects and handles URLs."""
+ mock_fetch.return_value = ("photo.png", b"photo data", "image/png")
+ mock_upload.return_value = "s3-key-xyz"
+ mock_client = MagicMock()
+
+ result = FileUploadable.from_path(
+ client=mock_client,
+ file="https://images.example.com/photo.png",
+ tool="SEND_EMAIL",
+ toolkit="gmail",
+ )
+
+ assert result.name == "photo.png"
+ assert result.mimetype == "image/png"
+ assert result.s3key == "s3-key-xyz"
+ mock_fetch.assert_called_once_with("https://images.example.com/photo.png")
+
+ @patch("composio.core.models._files._fetch_file_from_url")
+ def test_from_url_propagates_fetch_error(self, mock_fetch):
+ """Test that fetch errors are propagated correctly."""
+ mock_fetch.side_effect = ErrorUploadingFile("Fetch failed")
+ mock_client = MagicMock()
+
+ with pytest.raises(ErrorUploadingFile) as exc_info:
+ FileUploadable.from_url(
+ client=mock_client,
+ url="https://example.com/missing.jpg",
+ tool="TEST_TOOL",
+ toolkit="test_toolkit",
+ )
+
+ assert "Fetch failed" in str(exc_info.value)
+
+
+class TestFileHelperWithUrls:
+ """Test cases for FileHelper handling URLs in file uploads."""
+
+ @patch("composio.core.models._files.FileUploadable.from_path")
+ def test_substitute_file_uploads_with_url(
+ self, mock_from_path, file_helper, mock_tool
+ ):
+ """Test that URLs are correctly processed in substitute_file_uploads."""
+ mock_tool.input_parameters = {
+ "type": "object",
+ "properties": {
+ "attachment": {"type": "string", "file_uploadable": True},
+ },
+ }
+
+ mock_uploadable = MagicMock()
+ mock_uploadable.model_dump.return_value = {
+ "name": "image.jpg",
+ "mimetype": "image/jpeg",
+ "s3key": "s3-key-123",
+ }
+ mock_from_path.return_value = mock_uploadable
+
+ result = file_helper._substitute_file_uploads_recursively(
+ tool=mock_tool,
+ schema=mock_tool.input_parameters,
+ request={"attachment": "https://example.com/image.jpg"},
+ )
+
+ assert result["attachment"] == {
+ "name": "image.jpg",
+ "mimetype": "image/jpeg",
+ "s3key": "s3-key-123",
+ }
+ mock_from_path.assert_called_once()
+
+ @patch("composio.core.models._files.FileUploadable.from_path")
+ def test_substitute_file_uploads_with_url_in_anyof(
+ self, mock_from_path, file_helper, mock_tool
+ ):
+ """Test URL handling in anyOf schema variants."""
+ mock_tool.input_parameters = {
+ "type": "object",
+ "properties": {
+ "file": {
+ "anyOf": [
+ {"type": "string", "file_uploadable": True},
+ {"type": "null"},
+ ]
+ },
+ },
+ }
+
+ mock_uploadable = MagicMock()
+ mock_uploadable.model_dump.return_value = {
+ "name": "doc.pdf",
+ "mimetype": "application/pdf",
+ "s3key": "s3-key-456",
+ }
+ mock_from_path.return_value = mock_uploadable
+
+ result = file_helper._substitute_file_uploads_recursively(
+ tool=mock_tool,
+ schema=mock_tool.input_parameters,
+ request={"file": "https://docs.example.com/doc.pdf"},
+ )
+
+ assert result["file"] == {
+ "name": "doc.pdf",
+ "mimetype": "application/pdf",
+ "s3key": "s3-key-456",
+ }
+
+ @patch("composio.core.models._files.FileUploadable.from_path")
+ def test_substitute_file_uploads_array_with_urls(
+ self, mock_from_path, file_helper, mock_tool
+ ):
+ """Test URL handling in arrays with file_uploadable items."""
+ mock_tool.input_parameters = {
+ "type": "object",
+ "properties": {
+ "attachments": {
+ "type": "array",
+ "items": {"type": "string", "file_uploadable": True},
+ },
+ },
+ }
+
+ mock_uploadable1 = MagicMock()
+ mock_uploadable1.model_dump.return_value = {
+ "name": "file1.jpg",
+ "mimetype": "image/jpeg",
+ "s3key": "key1",
+ }
+ mock_uploadable2 = MagicMock()
+ mock_uploadable2.model_dump.return_value = {
+ "name": "file2.png",
+ "mimetype": "image/png",
+ "s3key": "key2",
+ }
+ mock_from_path.side_effect = [mock_uploadable1, mock_uploadable2]
+
+ result = file_helper._substitute_file_uploads_recursively(
+ tool=mock_tool,
+ schema=mock_tool.input_parameters,
+ request={
+ "attachments": [
+ "https://example.com/file1.jpg",
+ "https://example.com/file2.png",
+ ]
+ },
+ )
+
+ assert len(result["attachments"]) == 2
+ assert result["attachments"][0]["s3key"] == "key1"
+ assert result["attachments"][1]["s3key"] == "key2"