The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.
TypeScript aialternativeauthdatabasedenoembeddingsexamplefirebasenextjsoauth2pgvectorpostgispostgrespostgresqlpostgrestrealtimesupabasevectorswebsockets
fix(aeo): remove ua-based markdown serving (#47770)
Pamela Chia770f1c2b064d
diff --git a/apps/docs/app/api/guides-md/[...slug]/route.ts b/apps/docs/app/api/guides-md/[...slug]/route.tsindex 51dea28525c..f29c8258538 100644--- a/apps/docs/app/api/guides-md/[...slug]/route.ts+++ b/apps/docs/app/api/guides-md/[...slug]/route.ts@@ -18,6 +18,7 @@ export async function GET(request: Request, { params }: { params: Promise<{ slugheaders: {'Content-Type': 'text/markdown; charset=utf-8','Cache-Control': 'public, max-age=86400, stale-while-revalidate=3600',+ Vary: 'Accept',},})} catch {diff --git a/apps/docs/middleware.test.ts b/apps/docs/middleware.test.tsindex 9e447e4afb8..2c0c111e939 100644--- a/apps/docs/middleware.test.ts+++ b/apps/docs/middleware.test.ts@@ -84,14 +84,7 @@ describe('docs middleware — /guides/* content negotiation', () => {expect(res.headers.get('Vary')).toBe('Accept')})- it('does not 406 for LLM UAs or .md suffix paths even with a probe Accept', () => {- const llm = makeRequest('/docs/guides/auth', {- accept: 'application/x-content-negotiation-probe',- userAgent: 'Claude-User/1.0',- })- expect(middleware(llm).status).not.toBe(406)- expect(middleware(llm).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth'))-+ it('does not 406 for .md suffix paths even with a probe Accept', () => {const md = makeRequest('/docs/guides/auth.md', {accept: 'application/x-content-negotiation-probe',})@@ -99,6 +92,14 @@ describe('docs middleware — /guides/* content negotiation', () => {expect(middleware(md).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth'))})+ it('returns 406 for a probe Accept header regardless of user agent', () => {+ const req = makeRequest('/docs/guides/auth', {+ accept: 'application/x-content-negotiation-probe',+ userAgent: 'Claude-User/1.0',+ })+ expect(middleware(req).status).toBe(406)+ })+it('does not 406 on /reference/* (negotiation contract is /guides/* only)', () => {const req = makeRequest('/docs/reference/javascript/introduction', {accept: 'application/x-content-negotiation-probe',@@ -106,32 +107,40 @@ describe('docs middleware — /guides/* content negotiation', () => {expect(middleware(req).status).not.toBe(406)})- it('rewrites for each LLM user agent', () => {+ it('does not rewrite for agent user agents without a markdown Accept preference', () => {for (const ua of ['Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)','Claude-Web/1.0',- 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)','PerplexityBot/1.0',]) {const req = makeRequest('/docs/guides/auth', { userAgent: ua })- expect(middleware(req).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth'))+ expect(middleware(req).headers.get(REWRITE_HEADER)).toBeNull()}})- it('LLM UA overrides an Accept header that prefers HTML', () => {+ it('still serves ChatGPT-User markdown when Accept asks for it', () => {+ const req = makeRequest('/docs/guides/auth', {+ accept: 'text/markdown',+ userAgent: 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)',+ })+ expect(middleware(req).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth'))+ })++ it('serves HTML when Accept prefers HTML, regardless of user agent', () => {const req = makeRequest('/docs/guides/auth', {accept: 'text/html;q=1.0, text/markdown;q=0.1',userAgent: 'Claude-User/1.0',})- expect(middleware(req).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth'))+ expect(middleware(req).headers.get(REWRITE_HEADER)).toBeNull()})- it('falls through for non-LLM UAs (browsers, training crawlers, substring embeds)', () => {+ it('falls through for non-LLM UAs (browsers, training crawlers, excluded agents, substring embeds)', () => {for (const ua of ['Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36','GPTBot/1.0','ClaudeBot/1.0','CCBot/2.0',+ 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)','chatgpt-userscript/2.0','NotPerplexityBot',]) {diff --git a/apps/docs/middleware.ts b/apps/docs/middleware.tsindex 4bbeeed8a65..fd885725c23 100644--- a/apps/docs/middleware.ts+++ b/apps/docs/middleware.ts@@ -17,10 +17,7 @@ export function middleware(request: NextRequest) {const isMdSuffix = pathname.endsWith('.md')const slug = pathname.replace(`${GUIDES_PATH}/`, '').replace(/\.md$/, '')const decision = negotiateMarkdown(- {- acceptHeader: request.headers.get('accept') ?? '',- userAgent: request.headers.get('user-agent') ?? '',- },+ { acceptHeader: request.headers.get('accept') ?? '' },{ hasMarkdownVariant: GUIDES_MARKDOWN_SLUGS.has(slug), isMarkdownSuffix: isMdSuffix })diff --git a/apps/www/middleware.test.ts b/apps/www/middleware.test.tsindex dbeed5d70ec..6950e7c6c7c 100644--- a/apps/www/middleware.test.ts+++ b/apps/www/middleware.test.ts@@ -114,6 +114,30 @@ describe('www middleware', () => {expect(res.headers.get('x-middleware-rewrite')).toBeNull()})++ it('rewrites changelog entry .md requests without doubling the suffix', () => {+ const req = makeRequest('/changelog/100.md', { accept: 'text/markdown' })+ const res = middleware(req)++ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog/100.md')+ })++ it('serves markdown for explicit changelog .md requests even when Accept excludes it', () => {+ const req = makeRequest('/changelog/100.md', {+ accept: 'application/x-content-negotiation-probe',+ })+ const res = middleware(req)++ expect(res.status).not.toBe(406)+ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog/100.md')+ })++ it('rewrites the changelog index .md request without doubling the suffix', () => {+ const req = makeRequest('/changelog.md', { accept: 'text/markdown' })+ const res = middleware(req)++ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog.md')+ })})describe('Accept: text/markdown content negotiation', () => {@@ -153,6 +177,20 @@ describe('www middleware', () => {expect(res.headers.get('x-middleware-rewrite')).toBeNull()})++ it('rewrites changelog entries to their static .md file', () => {+ const req = makeRequest('/changelog/100', { accept: 'text/markdown' })+ const res = middleware(req)++ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog/100.md')+ })++ it('rewrites the bare changelog index to its static .md file', () => {+ const req = makeRequest('/changelog', { accept: 'text/markdown' })+ const res = middleware(req)++ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog.md')+ })})describe('Accept header q-value parsing', () => {@@ -234,15 +272,14 @@ describe('www middleware', () => {expect(res.status).not.toBe(406)})- it('does not return 406 for LLM UAs even with a probe Accept header', () => {+ it('returns 406 for a probe Accept header regardless of user agent', () => {const req = makeRequest('/pricing', {accept: 'application/x-content-negotiation-probe',userAgent: 'Claude-User/1.0',})const res = middleware(req)- expect(res.status).not.toBe(406)- expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing')+ expect(res.status).toBe(406)})it('returns 406 on changelog entries when Accept excludes every type', () => {@@ -264,39 +301,19 @@ describe('www middleware', () => {})})- describe('LLM user-agent routing', () => {- it('rewrites for Claude-User', () => {- const req = makeRequest('/pricing', {- userAgent: 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',- })- const res = middleware(req)-- expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing')- })-- it('rewrites for ChatGPT-User', () => {- const req = makeRequest('/auth', { userAgent: 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)' })- const res = middleware(req)-- expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')- })-- it('rewrites for Claude-Web', () => {- const req = makeRequest('/pricing', { userAgent: 'Claude-Web/1.0' })- const res = middleware(req)-- expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing')- })-- it('rewrites for PerplexityBot', () => {- const req = makeRequest('/auth/', { userAgent: 'PerplexityBot/1.0' })- const res = middleware(req)-- expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')- })-- it('falls through for UAs that embed a match as a substring', () => {- for (const ua of ['chatgpt-userscript/2.0', 'NotPerplexityBot']) {+ describe('user-agent independence', () => {+ it('serves HTML to agent and bot user agents that send no markdown Accept preference', () => {+ for (const ua of [+ 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',+ 'Claude-Web/1.0',+ 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)',+ 'PerplexityBot/1.0',+ 'GPTBot/1.0',+ 'ClaudeBot/1.0',+ 'CCBot/2.0',+ 'chatgpt-userscript/2.0',+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',+ ]) {const req = makeRequest('/auth', { userAgent: ua })const res = middleware(req)@@ -304,27 +321,18 @@ describe('www middleware', () => {}})- it('falls through for browser user agents', () => {+ it('negotiates by Accept as usual when an agent user agent is present', () => {const req = makeRequest('/auth', {- userAgent:- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',+ accept: 'text/markdown',+ userAgent: 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',})const res = middleware(req)- expect(res.headers.get('x-middleware-rewrite')).toBeNull()- })-- it('falls through for training crawlers (GPTBot, ClaudeBot, CCBot)', () => {- for (const ua of ['GPTBot/1.0', 'ClaudeBot/1.0', 'CCBot/2.0']) {- const req = makeRequest('/auth', { userAgent: ua })- const res = middleware(req)-- expect(res.headers.get('x-middleware-rewrite')).toBeNull()- }+ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')})it('falls through when slug is not in the allowlist', () => {- const req = makeRequest('/not-a-page', { userAgent: 'Claude-User' })+ const req = makeRequest('/not-a-page', { accept: 'text/markdown' })const res = middleware(req)expect(res.headers.get('x-middleware-rewrite')).toBeNull()diff --git a/apps/www/middleware.ts b/apps/www/middleware.tsindex cef09208638..ab96127d5ee 100644--- a/apps/www/middleware.ts+++ b/apps/www/middleware.ts@@ -7,25 +7,18 @@ import { MD_PAGES } from './app/api-v2/md/content.generated'export function middleware(request: NextRequest) {const { pathname } = request.nextUrl- if (pathname.endsWith('.md')) {- const slug = pathname.slice(1, -3)- if (MD_PAGES.has(slug)) {- return NextResponse.rewrite(new URL(`/api-v2/md/${slug}`, request.nextUrl))- }- }+ const isMarkdownSuffix = pathname.endsWith('.md')+ const basePathname = isMarkdownSuffix ? pathname.slice(0, -3) : pathname// Strip trailing slash so /auth/ and /auth resolve to the same allowlist// entry — NextURL preserves trailing-slash style on rewrite targets.- const slug = (pathname === '/' ? 'homepage' : pathname.slice(1)).replace(/\/$/, '')+ const slug = (basePathname === '/' ? 'homepage' : basePathname.slice(1)).replace(/\/$/, '')const isMdEligible = MD_PAGES.has(slug)const isChangelogEntry = slug === 'changelog' || /^changelog\/\d+/.test(slug)const decision = negotiateMarkdown(- {- acceptHeader: request.headers.get('accept') ?? '',- userAgent: request.headers.get('user-agent') ?? '',- },- { hasMarkdownVariant: isMdEligible || isChangelogEntry }+ { acceptHeader: request.headers.get('accept') ?? '' },+ { hasMarkdownVariant: isMdEligible || isChangelogEntry, isMarkdownSuffix })if (decision === 'not-acceptable') {diff --git a/packages/common/markdown-negotiation.test.ts b/packages/common/markdown-negotiation.test.tsindex cc43407abb9..8d18d77841c 100644--- a/packages/common/markdown-negotiation.test.ts+++ b/packages/common/markdown-negotiation.test.ts@@ -9,7 +9,7 @@ describe('negotiateMarkdown', () => {it('passes when the route has no markdown variant, regardless of other signals', () => {expect(negotiateMarkdown(- { acceptHeader: 'text/markdown', userAgent: 'Claude-User/1.0' },+ { acceptHeader: 'text/markdown' },{ hasMarkdownVariant: false, isMarkdownSuffix: true })).toBe('pass')@@ -17,60 +17,28 @@ describe('negotiateMarkdown', () => {})describe('forced markdown', () => {- it('returns markdown for LLM user agents even when Accept rejects everything', () => {+ it('returns markdown for an explicit .md suffix even with an HTML-only Accept', () => {expect(negotiateMarkdown(- { acceptHeader: 'application/x-content-negotiation-probe', userAgent: 'Claude-User/1.0' },- { hasMarkdownVariant: true }+ { acceptHeader: 'text/html' },+ { hasMarkdownVariant: true, isMarkdownSuffix: true })).toBe('markdown')})- it('returns markdown for an explicit .md suffix even with an HTML-only Accept', () => {+ it('returns markdown for an explicit .md suffix even when Accept rejects everything', () => {expect(negotiateMarkdown(- { acceptHeader: 'text/html', userAgent: '' },+ { acceptHeader: 'application/x-content-negotiation-probe' },{ hasMarkdownVariant: true, isMarkdownSuffix: true })).toBe('markdown')})-- it.each([- 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',- 'Claude-Web/1.0',- 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)',- 'PerplexityBot/1.0',- ])('treats %s as an LLM agent', (userAgent) => {- expect(negotiateMarkdown({ acceptHeader: '', userAgent }, { hasMarkdownVariant: true })).toBe(- 'markdown'- )- })-- it.each([- 'GPTBot/1.0',- 'ClaudeBot/1.0',- 'CCBot/2.0',- 'chatgpt-userscript/2.0',- 'NotPerplexityBot',- ])('does not treat %s (training crawler / substring embed) as an LLM agent', (userAgent) => {- expect(negotiateMarkdown({ acceptHeader: '', userAgent }, { hasMarkdownVariant: true })).toBe(- 'pass'- )- })-- it('caps user-agent length before matching', () => {- const padded = 'x'.repeat(600) + 'Claude-User'- expect(- negotiateMarkdown({ acceptHeader: '', userAgent: padded }, { hasMarkdownVariant: true })- ).toBe('pass')- })})describe('no Accept header', () => {it('passes (serves HTML) when no Accept header is sent', () => {- expect(- negotiateMarkdown({ acceptHeader: '', userAgent: '' }, { hasMarkdownVariant: true })- ).toBe('pass')+ expect(negotiateMarkdown({ acceptHeader: '' }, { hasMarkdownVariant: true })).toBe('pass')})})@@ -78,42 +46,34 @@ describe('negotiateMarkdown', () => {it('returns not-acceptable when Accept excludes every type we serve', () => {expect(negotiateMarkdown(- { acceptHeader: 'application/x-content-negotiation-probe', userAgent: '' },+ { acceptHeader: 'application/x-content-negotiation-probe' },{ hasMarkdownVariant: true })).toBe('not-acceptable')})it('does not 406 for bare */*', () => {- expect(- negotiateMarkdown({ acceptHeader: '*/*', userAgent: '' }, { hasMarkdownVariant: true })- ).toBe('pass')+ expect(negotiateMarkdown({ acceptHeader: '*/*' }, { hasMarkdownVariant: true })).toBe('pass')})})describe('q-value negotiation', () => {it('serves HTML for browser-style Accept', () => {expect(- negotiateMarkdown(- { acceptHeader: BROWSER_ACCEPT, userAgent: '' },- { hasMarkdownVariant: true }- )+ negotiateMarkdown({ acceptHeader: BROWSER_ACCEPT }, { hasMarkdownVariant: true })).toBe('pass')})it('serves markdown when explicitly requested', () => {expect(- negotiateMarkdown(- { acceptHeader: 'text/markdown', userAgent: '' },- { hasMarkdownVariant: true }- )+ negotiateMarkdown({ acceptHeader: 'text/markdown' }, { hasMarkdownVariant: true })).toBe('markdown')})it('serves markdown when its q-value beats html', () => {expect(negotiateMarkdown(- { acceptHeader: 'text/html;q=0.5, text/markdown;q=1.0', userAgent: '' },+ { acceptHeader: 'text/html;q=0.5, text/markdown;q=1.0' },{ hasMarkdownVariant: true })).toBe('markdown')@@ -122,7 +82,7 @@ describe('negotiateMarkdown', () => {it('serves HTML when its q-value beats markdown', () => {expect(negotiateMarkdown(- { acceptHeader: 'text/html;q=1.0, text/markdown;q=0.5', userAgent: '' },+ { acceptHeader: 'text/html;q=1.0, text/markdown;q=0.5' },{ hasMarkdownVariant: true })).toBe('pass')@@ -131,7 +91,7 @@ describe('negotiateMarkdown', () => {it('breaks an explicit md/html tie toward markdown', () => {expect(negotiateMarkdown(- { acceptHeader: 'text/markdown, text/html, */*', userAgent: '' },+ { acceptHeader: 'text/markdown, text/html, */*' },{ hasMarkdownVariant: true })).toBe('markdown')@@ -140,7 +100,7 @@ describe('negotiateMarkdown', () => {it('does not serve markdown when the client rejects it (q=0)', () => {expect(negotiateMarkdown(- { acceptHeader: 'text/markdown;q=0, text/html;q=1.0', userAgent: '' },+ { acceptHeader: 'text/markdown;q=0, text/html;q=1.0' },{ hasMarkdownVariant: true })).toBe('pass')@@ -149,7 +109,7 @@ describe('negotiateMarkdown', () => {it('tolerates OWS around the q parameter (RFC 9110)', () => {expect(negotiateMarkdown(- { acceptHeader: 'text/html ; q = 1.0, text/markdown ; q = 0.5', userAgent: '' },+ { acceptHeader: 'text/html ; q = 1.0, text/markdown ; q = 0.5' },{ hasMarkdownVariant: true })).toBe('pass')@@ -158,7 +118,7 @@ describe('negotiateMarkdown', () => {it('ignores out-of-range q-values (falls back to 1.0; tie -> markdown)', () => {expect(negotiateMarkdown(- { acceptHeader: 'text/html;q=2.0, text/markdown;q=1.0', userAgent: '' },+ { acceptHeader: 'text/html;q=2.0, text/markdown;q=1.0' },{ hasMarkdownVariant: true })).toBe('markdown')diff --git a/packages/common/markdown-negotiation.ts b/packages/common/markdown-negotiation.tsindex 28f8416815a..c9373d282e7 100644--- a/packages/common/markdown-negotiation.ts+++ b/packages/common/markdown-negotiation.ts@@ -1,17 +1,9 @@-// Live-fetch agents only. Training crawlers (GPTBot, ClaudeBot, CCBot) are-// governed by robots.txt; serving them content that differs from the HTML-// page risks SEO and cloaking penalties.-const LLM_USER_AGENT = /\bClaude-User\b|\bClaude-Web\b|\bChatGPT-User\b|\bPerplexityBot\b/i-// Media ranges (RFC 9110 §5.3.2) ordered most to least specific.const RANGES = ['text/markdown', 'text/html', 'text/*', '*/*'] as consttype Range = (typeof RANGES)[number]const Q_PARAM = /^\s*q\s*=\s*([\d.]+)\s*$/i-// Cap UA length before the regex test to bound CPU on the edge hot path.-const MAX_UA_LENGTH = 512-function isRange(s: string): s is Range {return (RANGES as readonly string[]).includes(s)}@@ -54,12 +46,17 @@ export type MarkdownDecision = 'markdown' | 'not-acceptable' | 'pass'/*** Content negotiation for routes that can serve either HTML or markdown.*- * `hasMarkdownVariant` is false for paths with no markdown representation (they- * never negotiate). `isMarkdownSuffix` forces markdown for an explicit `.md`- * request; callers that handle `.md` upstream can leave it false.+ * Markdown is served only on an explicit client signal: a `.md` request+ * (`isMarkdownSuffix`) or an Accept header preferring text/markdown. There is+ * deliberately no user-agent detection: responses that vary by UA poison+ * UA-blind CDN caches, and at least one major agent's reader hard-fails on+ * markdown it did not explicitly ask for.+ *+ * `hasMarkdownVariant` is false for paths with no markdown representation+ * (they never negotiate).*/export function negotiateMarkdown(- { acceptHeader, userAgent }: { acceptHeader: string; userAgent: string },+ { acceptHeader }: { acceptHeader: string },{hasMarkdownVariant,isMarkdownSuffix = false,@@ -67,18 +64,14 @@ export function negotiateMarkdown(): MarkdownDecision {if (!hasMarkdownVariant) return 'pass'- // LLM agents and an explicit `.md` request always get markdown.- if (LLM_USER_AGENT.test(userAgent.slice(0, MAX_UA_LENGTH)) || isMarkdownSuffix) {- return 'markdown'- }+ if (isMarkdownSuffix) return 'markdown'// No Accept header = browser/default client: serve HTML, never 406.if (!acceptHeader) return 'pass'const accept = parseAccept(acceptHeader)- // 406 when Accept rejects every type this route can produce. Only reached for- // non-LLM, non-`.md` clients that sent an Accept header (guards above), so a+ // 406 when Accept rejects every type this route can produce, so a// deliberate `Accept: application/json` gets a clean 406 instead of HTML.if (accept.markdown === 0 && accept.html === 0) return 'not-acceptable'