The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
feat(www): rewrite to md for known llm user agents (#45328)
Pamela Chiaa98a4928b47c
diff --git a/apps/www/middleware.test.ts b/apps/www/middleware.test.tsindex c6bcad36563..53310810985 100644--- a/apps/www/middleware.test.ts+++ b/apps/www/middleware.test.ts@@ -15,11 +15,17 @@ vi.mock('./app/api-v2/md/content.generated', () => ({function makeRequest(url: string,- { referer, hasCookie, accept }: { referer?: string; hasCookie?: boolean; accept?: string } = {}+ {+ referer,+ hasCookie,+ accept,+ userAgent,+ }: { referer?: string; hasCookie?: boolean; accept?: string; userAgent?: string } = {}): NextRequest {const headers: Record<string, string> = {}if (referer) headers.referer = refererif (accept) headers.accept = accept+ if (userAgent) headers['user-agent'] = userAgentconst req = new NextRequest(new URL(url, 'https://supabase.com'), { headers })if (hasCookie) {req.cookies.set(FIRST_REFERRER_COOKIE_NAME, 'existing')@@ -148,4 +154,71 @@ describe('www middleware', () => {expect(res.headers.get('x-middleware-rewrite')).toBeNull()})})++ describe('LLM user-agent routing', () => {+ it('rewrites for Claude-User', () => {+ const req = makeRequest('/pricing', {+ userAgent: 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',+ })+ const res = middleware(req)++ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing')+ })++ it('rewrites for ChatGPT-User', () => {+ const req = makeRequest('/auth', { userAgent: 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)' })+ const res = middleware(req)++ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')+ })++ it('rewrites for Claude-Web', () => {+ const req = makeRequest('/pricing', { userAgent: 'Claude-Web/1.0' })+ const res = middleware(req)++ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing')+ })++ it('rewrites for PerplexityBot', () => {+ const req = makeRequest('/auth/', { userAgent: 'PerplexityBot/1.0' })+ const res = middleware(req)++ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')+ })++ it('falls through for UAs that embed a match as a substring', () => {+ for (const ua of ['chatgpt-userscript/2.0', 'NotPerplexityBot']) {+ const req = makeRequest('/auth', { userAgent: ua })+ const res = middleware(req)++ expect(res.headers.get('x-middleware-rewrite')).toBeNull()+ }+ })++ it('falls through for browser user agents', () => {+ const req = makeRequest('/auth', {+ userAgent:+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',+ })+ const res = middleware(req)++ expect(res.headers.get('x-middleware-rewrite')).toBeNull()+ })++ it('falls through for training crawlers (GPTBot, ClaudeBot, CCBot)', () => {+ for (const ua of ['GPTBot/1.0', 'ClaudeBot/1.0', 'CCBot/2.0']) {+ const req = makeRequest('/auth', { userAgent: ua })+ const res = middleware(req)++ expect(res.headers.get('x-middleware-rewrite')).toBeNull()+ }+ })++ it('falls through when slug is not in the allowlist', () => {+ const req = makeRequest('/not-a-page', { userAgent: 'Claude-User' })+ const res = middleware(req)++ expect(res.headers.get('x-middleware-rewrite')).toBeNull()+ })+ })})diff --git a/apps/www/middleware.ts b/apps/www/middleware.tsindex bc4fb7cae3d..331f66a2db1 100644--- a/apps/www/middleware.ts+++ b/apps/www/middleware.ts@@ -3,6 +3,12 @@ import { NextResponse, type NextRequest } from 'next/server'import { MD_PAGES } from './app/api-v2/md/content.generated'+// Live-fetch LLM agents that retrieve pages on behalf of a user prompt.+// Training crawlers (GPTBot, CCBot, ClaudeBot, Anthropic-AI) are intentionally+// excluded; they are governed by robots.txt and serving them content that+// differs from the human HTML page would risk SEO and cloaking penalties.+const LLM_USER_AGENT = /\bClaude-User\b|\bClaude-Web\b|\bChatGPT-User\b|\bPerplexityBot\b/i+export function middleware(request: NextRequest) {const { pathname } = request.nextUrl@@ -14,9 +20,13 @@ export function middleware(request: NextRequest) {}}- // Content negotiation: Accept: text/markdown on known pages+ // Serve markdown to known LLM clients (Accept header or UA match).+ // Cache-key safety: rewriting to /api-v2/md/<slug> partitions the response+ // by path, so no Vary: User-Agent is needed.const accept = (request.headers.get('accept') ?? '').toLowerCase()- if (accept.includes('text/markdown')) {+ // Cap UA length before regex test to bound CPU on the edge hot path.+ const userAgent = (request.headers.get('user-agent') ?? '').slice(0, 512)+ if (accept.includes('text/markdown') || LLM_USER_AGENT.test(userAgent)) {// Strip trailing slash so /auth/ and /auth resolve to the same allowlist entry.// (NextURL's pathname setter preserves the trailing-slash style of the cloned// origin, which would otherwise leak through to the rewrite target.)