Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): Increase max_field_size in aiohttp requests (#11261)

seer-by-sentry[bot]3dc5208f7166
diff --git a/autogpt_platform/backend/backend/util/request.py b/autogpt_platform/backend/backend/util/request.py
index 298beec5f0..6e5c6a46ae 100644
--- a/autogpt_platform/backend/backend/util/request.py
+++ b/autogpt_platform/backend/backend/util/request.py
@@ -425,6 +425,9 @@ class Requests:
req_headers["Host"] = hostname
# Override data if files are provided
+ # Set max_field_size to handle servers with large headers (e.g., long CSP headers)
+ # Default is 8190 bytes, we increase to 16KB to accommodate legitimate large headers
+ session_kwargs["max_field_size"] = 16384
async with aiohttp.ClientSession(**session_kwargs) as session:
# Perform the request with redirects disabled for manual handling
diff --git a/autogpt_platform/backend/backend/util/request_test.py b/autogpt_platform/backend/backend/util/request_test.py
index 57717ff77f..eaabbf4e09 100644
--- a/autogpt_platform/backend/backend/util/request_test.py
+++ b/autogpt_platform/backend/backend/util/request_test.py
@@ -1,4 +1,5 @@
import pytest
+from aiohttp import web
from backend.util.request import pin_url, validate_url
@@ -110,3 +111,63 @@ async def test_dns_rebinding_fix(
assert expected_ip in pinned_url
# The unpinned URL's hostname should match our original IDNA encoded hostname
assert url.hostname == hostname
+
+
+async def test_large_header_handling():
+ """Test that ClientSession with max_field_size=16384 can handle large headers (>8190 bytes)"""
+ import aiohttp
+
+ # Create a test server that returns large headers
+ async def large_header_handler(request):
+ # Create a header value larger than the default aiohttp max_field_size (8190 bytes)
+ # Simulate a long CSP header or similar legitimate large header
+ large_value = "policy-" + "x" * 8500
+ return web.Response(
+ text="OK",
+ headers={"X-Large-Header": large_value},
+ )
+
+ app = web.Application()
+ app.router.add_get("/large-header", large_header_handler)
+
+ # Start test server
+ runner = web.AppRunner(app)
+ await runner.setup()
+ site = web.TCPSite(runner, "127.0.0.1", 0)
+ await site.start()
+
+ try:
+ # Get the port from the server
+ server = site._server
+ assert server is not None
+ sockets = getattr(server, "sockets", None)
+ assert sockets is not None
+ port = sockets[0].getsockname()[1]
+
+ # Test with default max_field_size (should fail)
+ default_failed = False
+ try:
+ async with aiohttp.ClientSession() as session:
+ async with session.get(f"http://127.0.0.1:{port}/large-header") as resp:
+ await resp.read()
+ except Exception:
+ # Expected: any error with default settings when header > 8190 bytes
+ default_failed = True
+
+ assert default_failed, "Expected error with default max_field_size"
+
+ # Test with increased max_field_size (should succeed)
+ # This is the fix: setting max_field_size=16384 allows headers up to 16KB
+ async with aiohttp.ClientSession(max_field_size=16384) as session:
+ async with session.get(f"http://127.0.0.1:{port}/large-header") as resp:
+ body = await resp.read()
+ # Verify the response is successful
+ assert resp.status == 200
+ assert "X-Large-Header" in resp.headers
+ # Verify the large header value was received
+ assert len(resp.headers["X-Large-Header"]) > 8190
+ assert body == b"OK"
+
+ finally:
+ await runner.cleanup()