Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

feat(www): rewrite to md for known llm user agents (#45328)

Pamela Chiaa98a4928b47c
diff --git a/apps/www/middleware.test.ts b/apps/www/middleware.test.ts
index c6bcad36563..53310810985 100644
--- a/apps/www/middleware.test.ts
+++ b/apps/www/middleware.test.ts
@@ -15,11 +15,17 @@ vi.mock('./app/api-v2/md/content.generated', () => ({
function makeRequest(
url: string,
- { referer, hasCookie, accept }: { referer?: string; hasCookie?: boolean; accept?: string } = {}
+ {
+ referer,
+ hasCookie,
+ accept,
+ userAgent,
+ }: { referer?: string; hasCookie?: boolean; accept?: string; userAgent?: string } = {}
): NextRequest {
const headers: Record<string, string> = {}
if (referer) headers.referer = referer
if (accept) headers.accept = accept
+ if (userAgent) headers['user-agent'] = userAgent
const req = new NextRequest(new URL(url, 'https://supabase.com'), { headers })
if (hasCookie) {
req.cookies.set(FIRST_REFERRER_COOKIE_NAME, 'existing')
@@ -148,4 +154,71 @@ describe('www middleware', () => {
expect(res.headers.get('x-middleware-rewrite')).toBeNull()
})
})
+
+ describe('LLM user-agent routing', () => {
+ it('rewrites for Claude-User', () => {
+ const req = makeRequest('/pricing', {
+ userAgent: 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',
+ })
+ const res = middleware(req)
+
+ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing')
+ })
+
+ it('rewrites for ChatGPT-User', () => {
+ const req = makeRequest('/auth', { userAgent: 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)' })
+ const res = middleware(req)
+
+ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')
+ })
+
+ it('rewrites for Claude-Web', () => {
+ const req = makeRequest('/pricing', { userAgent: 'Claude-Web/1.0' })
+ const res = middleware(req)
+
+ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing')
+ })
+
+ it('rewrites for PerplexityBot', () => {
+ const req = makeRequest('/auth/', { userAgent: 'PerplexityBot/1.0' })
+ const res = middleware(req)
+
+ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')
+ })
+
+ it('falls through for UAs that embed a match as a substring', () => {
+ for (const ua of ['chatgpt-userscript/2.0', 'NotPerplexityBot']) {
+ const req = makeRequest('/auth', { userAgent: ua })
+ const res = middleware(req)
+
+ expect(res.headers.get('x-middleware-rewrite')).toBeNull()
+ }
+ })
+
+ it('falls through for browser user agents', () => {
+ const req = makeRequest('/auth', {
+ userAgent:
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+ })
+ const res = middleware(req)
+
+ expect(res.headers.get('x-middleware-rewrite')).toBeNull()
+ })
+
+ it('falls through for training crawlers (GPTBot, ClaudeBot, CCBot)', () => {
+ for (const ua of ['GPTBot/1.0', 'ClaudeBot/1.0', 'CCBot/2.0']) {
+ const req = makeRequest('/auth', { userAgent: ua })
+ const res = middleware(req)
+
+ expect(res.headers.get('x-middleware-rewrite')).toBeNull()
+ }
+ })
+
+ it('falls through when slug is not in the allowlist', () => {
+ const req = makeRequest('/not-a-page', { userAgent: 'Claude-User' })
+ const res = middleware(req)
+
+ expect(res.headers.get('x-middleware-rewrite')).toBeNull()
+ })
+ })
})
diff --git a/apps/www/middleware.ts b/apps/www/middleware.ts
index bc4fb7cae3d..331f66a2db1 100644
--- a/apps/www/middleware.ts
+++ b/apps/www/middleware.ts
@@ -3,6 +3,12 @@ import { NextResponse, type NextRequest } from 'next/server'
import { MD_PAGES } from './app/api-v2/md/content.generated'
+// Live-fetch LLM agents that retrieve pages on behalf of a user prompt.
+// Training crawlers (GPTBot, CCBot, ClaudeBot, Anthropic-AI) are intentionally
+// excluded; they are governed by robots.txt and serving them content that
+// differs from the human HTML page would risk SEO and cloaking penalties.
+const LLM_USER_AGENT = /\bClaude-User\b|\bClaude-Web\b|\bChatGPT-User\b|\bPerplexityBot\b/i
+
export function middleware(request: NextRequest) {
const { pathname } = request.nextUrl
@@ -14,9 +20,13 @@ export function middleware(request: NextRequest) {
}
}
- // Content negotiation: Accept: text/markdown on known pages
+ // Serve markdown to known LLM clients (Accept header or UA match).
+ // Cache-key safety: rewriting to /api-v2/md/<slug> partitions the response
+ // by path, so no Vary: User-Agent is needed.
const accept = (request.headers.get('accept') ?? '').toLowerCase()
- if (accept.includes('text/markdown')) {
+ // Cap UA length before regex test to bound CPU on the edge hot path.
+ const userAgent = (request.headers.get('user-agent') ?? '').slice(0, 512)
+ if (accept.includes('text/markdown') || LLM_USER_AGENT.test(userAgent)) {
// Strip trailing slash so /auth/ and /auth resolve to the same allowlist entry.
// (NextURL's pathname setter preserves the trailing-slash style of the cloned
// origin, which would otherwise leak through to the rewrite target.)