Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

fix(ai-gateway): remove Free chat context ceiling (#5356)

Louis Beaumont0aa26fbf3f4c
diff --git a/packages/ai-gateway/src/services/free-chat-limit.test.ts b/packages/ai-gateway/src/services/free-chat-limit.test.ts
index c820cf222..640f03faa 100644
--- a/packages/ai-gateway/src/services/free-chat-limit.test.ts
+++ b/packages/ai-gateway/src/services/free-chat-limit.test.ts
@@ -11,13 +11,11 @@ import {
FREE_CHAT_MESSAGE_LIMIT,
FREE_CHAT_MAX_IMAGE_BYTES,
FREE_CHAT_MAX_IMAGES,
- FREE_CHAT_MAX_MESSAGES,
FREE_CHAT_MAX_PROVIDER_CALLS_PER_MESSAGE,
FREE_CHAT_MAX_OUTPUT_TOKENS,
FREE_CHAT_MAX_REQUEST_BYTES,
FREE_CHAT_MAX_RESPONSE_FORMAT_BYTES,
FREE_CHAT_MAX_STRUCTURE_DEPTH,
- FREE_CHAT_MAX_TEXT_BYTES,
FREE_CHAT_MAX_TOOLS,
FREE_CHAT_MAX_TOOLS_BYTES,
acquireFreeChatLease,
@@ -478,18 +476,18 @@ describe('validateFreeChatRequestLimits', () => {
expect(error?.code).toBe('free_chat_request_too_large');
});
- it('caps message count and aggregate text/tool-result bytes', () => {
- const tooMany = validateFreeChatRequestLimits(
- bodyWith(Array.from({ length: FREE_CHAT_MAX_MESSAGES + 1 }, () => ({ role: 'user', content: 'x' }))),
- preview,
- );
- expect(tooMany?.code).toBe('free_chat_too_many_messages');
+ it('does not impose a Free-plan aggregate-text or message-count ceiling', () => {
+ const body = bodyWith([
+ ...Array.from({ length: 128 }, () => ({ role: 'user', content: 'continue' })),
+ {
+ role: 'assistant',
+ content: '',
+ tool_calls: [{ id: 'call_1', type: 'function', function: { name: 'search', arguments: '{}' } }],
+ },
+ { role: 'tool', content: 'x'.repeat(5 * 1024 * 1024), tool_call_id: 'call_1' },
+ ]);
- const tooMuchText = validateFreeChatRequestLimits(
- bodyWith([{ role: 'user', content: 'x'.repeat(FREE_CHAT_MAX_TEXT_BYTES + 1) }]),
- preview,
- );
- expect(tooMuchText?.code).toBe('free_chat_text_too_large');
+ expect(validateFreeChatRequestLimits(body, preview)).toBeNull();
});
it('caps encoded image size and image count independently', () => {
@@ -524,16 +522,16 @@ describe('validateFreeChatRequestLimits', () => {
expect(alternateSource?.code).toBe('free_chat_image_too_large');
});
- it('does not let mixed image fields disguise text from the text-byte cap', () => {
- const disguisedText = validateFreeChatRequestLimits(bodyWith([{
+ it('treats mixed text fields as context rather than image payloads', () => {
+ const mixedText = validateFreeChatRequestLimits(bodyWith([{
role: 'user',
content: [{
type: 'text',
- text: 'x'.repeat(FREE_CHAT_MAX_TEXT_BYTES + 1),
+ text: 'x'.repeat(1024 * 1024),
image_url: { url: 'data:image/png;base64,YQ==' },
} as any],
}]), preview);
- expect(disguisedText?.code).toBe('free_chat_text_too_large');
+ expect(mixedText).toBeNull();
});
it('rejects non-image media disguised with an image content type', () => {
diff --git a/packages/ai-gateway/src/services/free-chat-limit.ts b/packages/ai-gateway/src/services/free-chat-limit.ts
index feaef1720..488192658 100644
--- a/packages/ai-gateway/src/services/free-chat-limit.ts
+++ b/packages/ai-gateway/src/services/free-chat-limit.ts
@@ -9,9 +9,7 @@ export const FREE_CHAT_MESSAGE_LIMIT = 2;
export const FREE_CHAT_MAX_PROVIDER_CALLS_PER_MESSAGE = 8;
export const FREE_CHAT_MAX_OUTPUT_TOKENS = 4096;
export const FREE_CHAT_MAX_REQUEST_BYTES = 8 * 1024 * 1024;
-export const FREE_CHAT_MAX_MESSAGES = 96;
export const FREE_CHAT_MAX_MESSAGE_BYTES = 6 * 1024 * 1024;
-export const FREE_CHAT_MAX_TEXT_BYTES = 64 * 1024;
export const FREE_CHAT_MAX_IMAGES = 4;
export const FREE_CHAT_MAX_IMAGE_BYTES = 2 * 1024 * 1024;
export const FREE_CHAT_MAX_TOOLS = 48;
@@ -22,7 +20,7 @@ export const FREE_CHAT_MAX_IN_FLIGHT = 1;
export const FREE_CHAT_IN_FLIGHT_LEASE_SECONDS = 10 * 60;
// This is a conservative reservation for both entries in the dedicated
-// preview waterfall at the maximum text/tool/output limits above. Reserving
+// preview waterfall at the maximum request/tool/output limits above. Reserving
// before inference makes concurrent requests unable to race a post-hoc spend
// check. Keep this in sync with FREE_PREVIEW_WATERFALL in handlers/chat.ts.
export const FREE_CHAT_COST_RESERVATION_MICRO_USD = 150_000;
@@ -212,14 +210,7 @@ export function validateFreeChatRequestBodyLimits(
if (!Array.isArray(body.messages)) {
return { status: 400, code: 'invalid_free_chat_messages', message: 'Free hosted chat requires a messages array.' };
}
- if (body.messages.length > FREE_CHAT_MAX_MESSAGES) {
- return requestTooLarge(
- 'free_chat_too_many_messages',
- `Free hosted chat supports at most ${FREE_CHAT_MAX_MESSAGES} messages per request.`,
- );
- }
- let textBytes = 0;
let imageCount = 0;
for (const message of body.messages) {
if (!message || typeof message !== 'object') {
@@ -252,11 +243,8 @@ export function validateFreeChatRequestBodyLimits(
);
}
- const { content, ...messageMetadata } = message;
- textBytes += jsonByteLength(messageMetadata);
- if (typeof content === 'string') {
- textBytes += byteLength(content);
- } else if (Array.isArray(content)) {
+ const { content } = message;
+ if (Array.isArray(content)) {
for (const part of content) {
if (!part || typeof part !== 'object') {
return {
@@ -280,21 +268,11 @@ export function validateFreeChatRequestBodyLimits(
`Each free hosted chat image is limited to ${FREE_CHAT_MAX_IMAGE_BYTES} encoded bytes.`,
);
}
- } else {
- textBytes += jsonByteLength(part);
}
}
- } else if (content !== null && content !== undefined) {
- textBytes += jsonByteLength(content);
}
}
- if (textBytes > FREE_CHAT_MAX_TEXT_BYTES) {
- return requestTooLarge(
- 'free_chat_text_too_large',
- `Free hosted chat text and tool results are limited to ${FREE_CHAT_MAX_TEXT_BYTES} bytes per request.`,
- );
- }
if (imageCount > FREE_CHAT_MAX_IMAGES) {
return requestTooLarge(
'free_chat_too_many_images',