Skip to content

The Postgres development platform. Supabase gives you a dedicated Postgres database to build your web, mobile, and AI applications.

fix(aeo): remove ua-based markdown serving (#47770)

Pamela Chia770f1c2b064d
diff --git a/apps/docs/app/api/guides-md/[...slug]/route.ts b/apps/docs/app/api/guides-md/[...slug]/route.ts
index 51dea28525c..f29c8258538 100644
--- a/apps/docs/app/api/guides-md/[...slug]/route.ts
+++ b/apps/docs/app/api/guides-md/[...slug]/route.ts
@@ -18,6 +18,7 @@ export async function GET(request: Request, { params }: { params: Promise<{ slug
headers: {
'Content-Type': 'text/markdown; charset=utf-8',
'Cache-Control': 'public, max-age=86400, stale-while-revalidate=3600',
+ Vary: 'Accept',
},
})
} catch {
diff --git a/apps/docs/middleware.test.ts b/apps/docs/middleware.test.ts
index 9e447e4afb8..2c0c111e939 100644
--- a/apps/docs/middleware.test.ts
+++ b/apps/docs/middleware.test.ts
@@ -84,14 +84,7 @@ describe('docs middleware — /guides/* content negotiation', () => {
expect(res.headers.get('Vary')).toBe('Accept')
})
- it('does not 406 for LLM UAs or .md suffix paths even with a probe Accept', () => {
- const llm = makeRequest('/docs/guides/auth', {
- accept: 'application/x-content-negotiation-probe',
- userAgent: 'Claude-User/1.0',
- })
- expect(middleware(llm).status).not.toBe(406)
- expect(middleware(llm).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth'))
-
+ it('does not 406 for .md suffix paths even with a probe Accept', () => {
const md = makeRequest('/docs/guides/auth.md', {
accept: 'application/x-content-negotiation-probe',
})
@@ -99,6 +92,14 @@ describe('docs middleware — /guides/* content negotiation', () => {
expect(middleware(md).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth'))
})
+ it('returns 406 for a probe Accept header regardless of user agent', () => {
+ const req = makeRequest('/docs/guides/auth', {
+ accept: 'application/x-content-negotiation-probe',
+ userAgent: 'Claude-User/1.0',
+ })
+ expect(middleware(req).status).toBe(406)
+ })
+
it('does not 406 on /reference/* (negotiation contract is /guides/* only)', () => {
const req = makeRequest('/docs/reference/javascript/introduction', {
accept: 'application/x-content-negotiation-probe',
@@ -106,32 +107,40 @@ describe('docs middleware — /guides/* content negotiation', () => {
expect(middleware(req).status).not.toBe(406)
})
- it('rewrites for each LLM user agent', () => {
+ it('does not rewrite for agent user agents without a markdown Accept preference', () => {
for (const ua of [
'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',
'Claude-Web/1.0',
- 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)',
'PerplexityBot/1.0',
]) {
const req = makeRequest('/docs/guides/auth', { userAgent: ua })
- expect(middleware(req).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth'))
+ expect(middleware(req).headers.get(REWRITE_HEADER)).toBeNull()
}
})
- it('LLM UA overrides an Accept header that prefers HTML', () => {
+ it('still serves ChatGPT-User markdown when Accept asks for it', () => {
+ const req = makeRequest('/docs/guides/auth', {
+ accept: 'text/markdown',
+ userAgent: 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)',
+ })
+ expect(middleware(req).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth'))
+ })
+
+ it('serves HTML when Accept prefers HTML, regardless of user agent', () => {
const req = makeRequest('/docs/guides/auth', {
accept: 'text/html;q=1.0, text/markdown;q=0.1',
userAgent: 'Claude-User/1.0',
})
- expect(middleware(req).headers.get(REWRITE_HEADER)).toBe(GUIDES_MD_REWRITE('auth'))
+ expect(middleware(req).headers.get(REWRITE_HEADER)).toBeNull()
})
- it('falls through for non-LLM UAs (browsers, training crawlers, substring embeds)', () => {
+ it('falls through for non-LLM UAs (browsers, training crawlers, excluded agents, substring embeds)', () => {
for (const ua of [
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
'GPTBot/1.0',
'ClaudeBot/1.0',
'CCBot/2.0',
+ 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)',
'chatgpt-userscript/2.0',
'NotPerplexityBot',
]) {
diff --git a/apps/docs/middleware.ts b/apps/docs/middleware.ts
index 4bbeeed8a65..fd885725c23 100644
--- a/apps/docs/middleware.ts
+++ b/apps/docs/middleware.ts
@@ -17,10 +17,7 @@ export function middleware(request: NextRequest) {
const isMdSuffix = pathname.endsWith('.md')
const slug = pathname.replace(`${GUIDES_PATH}/`, '').replace(/\.md$/, '')
const decision = negotiateMarkdown(
- {
- acceptHeader: request.headers.get('accept') ?? '',
- userAgent: request.headers.get('user-agent') ?? '',
- },
+ { acceptHeader: request.headers.get('accept') ?? '' },
{ hasMarkdownVariant: GUIDES_MARKDOWN_SLUGS.has(slug), isMarkdownSuffix: isMdSuffix }
)
diff --git a/apps/www/middleware.test.ts b/apps/www/middleware.test.ts
index dbeed5d70ec..6950e7c6c7c 100644
--- a/apps/www/middleware.test.ts
+++ b/apps/www/middleware.test.ts
@@ -114,6 +114,30 @@ describe('www middleware', () => {
expect(res.headers.get('x-middleware-rewrite')).toBeNull()
})
+
+ it('rewrites changelog entry .md requests without doubling the suffix', () => {
+ const req = makeRequest('/changelog/100.md', { accept: 'text/markdown' })
+ const res = middleware(req)
+
+ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog/100.md')
+ })
+
+ it('serves markdown for explicit changelog .md requests even when Accept excludes it', () => {
+ const req = makeRequest('/changelog/100.md', {
+ accept: 'application/x-content-negotiation-probe',
+ })
+ const res = middleware(req)
+
+ expect(res.status).not.toBe(406)
+ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog/100.md')
+ })
+
+ it('rewrites the changelog index .md request without doubling the suffix', () => {
+ const req = makeRequest('/changelog.md', { accept: 'text/markdown' })
+ const res = middleware(req)
+
+ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog.md')
+ })
})
describe('Accept: text/markdown content negotiation', () => {
@@ -153,6 +177,20 @@ describe('www middleware', () => {
expect(res.headers.get('x-middleware-rewrite')).toBeNull()
})
+
+ it('rewrites changelog entries to their static .md file', () => {
+ const req = makeRequest('/changelog/100', { accept: 'text/markdown' })
+ const res = middleware(req)
+
+ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog/100.md')
+ })
+
+ it('rewrites the bare changelog index to its static .md file', () => {
+ const req = makeRequest('/changelog', { accept: 'text/markdown' })
+ const res = middleware(req)
+
+ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/changelog.md')
+ })
})
describe('Accept header q-value parsing', () => {
@@ -234,15 +272,14 @@ describe('www middleware', () => {
expect(res.status).not.toBe(406)
})
- it('does not return 406 for LLM UAs even with a probe Accept header', () => {
+ it('returns 406 for a probe Accept header regardless of user agent', () => {
const req = makeRequest('/pricing', {
accept: 'application/x-content-negotiation-probe',
userAgent: 'Claude-User/1.0',
})
const res = middleware(req)
- expect(res.status).not.toBe(406)
- expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing')
+ expect(res.status).toBe(406)
})
it('returns 406 on changelog entries when Accept excludes every type', () => {
@@ -264,39 +301,19 @@ describe('www middleware', () => {
})
})
- describe('LLM user-agent routing', () => {
- it('rewrites for Claude-User', () => {
- const req = makeRequest('/pricing', {
- userAgent: 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',
- })
- const res = middleware(req)
-
- expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing')
- })
-
- it('rewrites for ChatGPT-User', () => {
- const req = makeRequest('/auth', { userAgent: 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)' })
- const res = middleware(req)
-
- expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')
- })
-
- it('rewrites for Claude-Web', () => {
- const req = makeRequest('/pricing', { userAgent: 'Claude-Web/1.0' })
- const res = middleware(req)
-
- expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/pricing')
- })
-
- it('rewrites for PerplexityBot', () => {
- const req = makeRequest('/auth/', { userAgent: 'PerplexityBot/1.0' })
- const res = middleware(req)
-
- expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')
- })
-
- it('falls through for UAs that embed a match as a substring', () => {
- for (const ua of ['chatgpt-userscript/2.0', 'NotPerplexityBot']) {
+ describe('user-agent independence', () => {
+ it('serves HTML to agent and bot user agents that send no markdown Accept preference', () => {
+ for (const ua of [
+ 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',
+ 'Claude-Web/1.0',
+ 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)',
+ 'PerplexityBot/1.0',
+ 'GPTBot/1.0',
+ 'ClaudeBot/1.0',
+ 'CCBot/2.0',
+ 'chatgpt-userscript/2.0',
+ 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+ ]) {
const req = makeRequest('/auth', { userAgent: ua })
const res = middleware(req)
@@ -304,27 +321,18 @@ describe('www middleware', () => {
}
})
- it('falls through for browser user agents', () => {
+ it('negotiates by Accept as usual when an agent user agent is present', () => {
const req = makeRequest('/auth', {
- userAgent:
- 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+ accept: 'text/markdown',
+ userAgent: 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',
})
const res = middleware(req)
- expect(res.headers.get('x-middleware-rewrite')).toBeNull()
- })
-
- it('falls through for training crawlers (GPTBot, ClaudeBot, CCBot)', () => {
- for (const ua of ['GPTBot/1.0', 'ClaudeBot/1.0', 'CCBot/2.0']) {
- const req = makeRequest('/auth', { userAgent: ua })
- const res = middleware(req)
-
- expect(res.headers.get('x-middleware-rewrite')).toBeNull()
- }
+ expect(res.headers.get('x-middleware-rewrite')).toBe('https://supabase.com/api-v2/md/auth')
})
it('falls through when slug is not in the allowlist', () => {
- const req = makeRequest('/not-a-page', { userAgent: 'Claude-User' })
+ const req = makeRequest('/not-a-page', { accept: 'text/markdown' })
const res = middleware(req)
expect(res.headers.get('x-middleware-rewrite')).toBeNull()
diff --git a/apps/www/middleware.ts b/apps/www/middleware.ts
index cef09208638..ab96127d5ee 100644
--- a/apps/www/middleware.ts
+++ b/apps/www/middleware.ts
@@ -7,25 +7,18 @@ import { MD_PAGES } from './app/api-v2/md/content.generated'
export function middleware(request: NextRequest) {
const { pathname } = request.nextUrl
- if (pathname.endsWith('.md')) {
- const slug = pathname.slice(1, -3)
- if (MD_PAGES.has(slug)) {
- return NextResponse.rewrite(new URL(`/api-v2/md/${slug}`, request.nextUrl))
- }
- }
+ const isMarkdownSuffix = pathname.endsWith('.md')
+ const basePathname = isMarkdownSuffix ? pathname.slice(0, -3) : pathname
// Strip trailing slash so /auth/ and /auth resolve to the same allowlist
// entry — NextURL preserves trailing-slash style on rewrite targets.
- const slug = (pathname === '/' ? 'homepage' : pathname.slice(1)).replace(/\/$/, '')
+ const slug = (basePathname === '/' ? 'homepage' : basePathname.slice(1)).replace(/\/$/, '')
const isMdEligible = MD_PAGES.has(slug)
const isChangelogEntry = slug === 'changelog' || /^changelog\/\d+/.test(slug)
const decision = negotiateMarkdown(
- {
- acceptHeader: request.headers.get('accept') ?? '',
- userAgent: request.headers.get('user-agent') ?? '',
- },
- { hasMarkdownVariant: isMdEligible || isChangelogEntry }
+ { acceptHeader: request.headers.get('accept') ?? '' },
+ { hasMarkdownVariant: isMdEligible || isChangelogEntry, isMarkdownSuffix }
)
if (decision === 'not-acceptable') {
diff --git a/packages/common/markdown-negotiation.test.ts b/packages/common/markdown-negotiation.test.ts
index cc43407abb9..8d18d77841c 100644
--- a/packages/common/markdown-negotiation.test.ts
+++ b/packages/common/markdown-negotiation.test.ts
@@ -9,7 +9,7 @@ describe('negotiateMarkdown', () => {
it('passes when the route has no markdown variant, regardless of other signals', () => {
expect(
negotiateMarkdown(
- { acceptHeader: 'text/markdown', userAgent: 'Claude-User/1.0' },
+ { acceptHeader: 'text/markdown' },
{ hasMarkdownVariant: false, isMarkdownSuffix: true }
)
).toBe('pass')
@@ -17,60 +17,28 @@ describe('negotiateMarkdown', () => {
})
describe('forced markdown', () => {
- it('returns markdown for LLM user agents even when Accept rejects everything', () => {
+ it('returns markdown for an explicit .md suffix even with an HTML-only Accept', () => {
expect(
negotiateMarkdown(
- { acceptHeader: 'application/x-content-negotiation-probe', userAgent: 'Claude-User/1.0' },
- { hasMarkdownVariant: true }
+ { acceptHeader: 'text/html' },
+ { hasMarkdownVariant: true, isMarkdownSuffix: true }
)
).toBe('markdown')
})
- it('returns markdown for an explicit .md suffix even with an HTML-only Accept', () => {
+ it('returns markdown for an explicit .md suffix even when Accept rejects everything', () => {
expect(
negotiateMarkdown(
- { acceptHeader: 'text/html', userAgent: '' },
+ { acceptHeader: 'application/x-content-negotiation-probe' },
{ hasMarkdownVariant: true, isMarkdownSuffix: true }
)
).toBe('markdown')
})
-
- it.each([
- 'Claude-User (claude-code/2.1.119; +https://support.anthropic.com/)',
- 'Claude-Web/1.0',
- 'Mozilla/5.0 (compatible; ChatGPT-User/1.0)',
- 'PerplexityBot/1.0',
- ])('treats %s as an LLM agent', (userAgent) => {
- expect(negotiateMarkdown({ acceptHeader: '', userAgent }, { hasMarkdownVariant: true })).toBe(
- 'markdown'
- )
- })
-
- it.each([
- 'GPTBot/1.0',
- 'ClaudeBot/1.0',
- 'CCBot/2.0',
- 'chatgpt-userscript/2.0',
- 'NotPerplexityBot',
- ])('does not treat %s (training crawler / substring embed) as an LLM agent', (userAgent) => {
- expect(negotiateMarkdown({ acceptHeader: '', userAgent }, { hasMarkdownVariant: true })).toBe(
- 'pass'
- )
- })
-
- it('caps user-agent length before matching', () => {
- const padded = 'x'.repeat(600) + 'Claude-User'
- expect(
- negotiateMarkdown({ acceptHeader: '', userAgent: padded }, { hasMarkdownVariant: true })
- ).toBe('pass')
- })
})
describe('no Accept header', () => {
it('passes (serves HTML) when no Accept header is sent', () => {
- expect(
- negotiateMarkdown({ acceptHeader: '', userAgent: '' }, { hasMarkdownVariant: true })
- ).toBe('pass')
+ expect(negotiateMarkdown({ acceptHeader: '' }, { hasMarkdownVariant: true })).toBe('pass')
})
})
@@ -78,42 +46,34 @@ describe('negotiateMarkdown', () => {
it('returns not-acceptable when Accept excludes every type we serve', () => {
expect(
negotiateMarkdown(
- { acceptHeader: 'application/x-content-negotiation-probe', userAgent: '' },
+ { acceptHeader: 'application/x-content-negotiation-probe' },
{ hasMarkdownVariant: true }
)
).toBe('not-acceptable')
})
it('does not 406 for bare */*', () => {
- expect(
- negotiateMarkdown({ acceptHeader: '*/*', userAgent: '' }, { hasMarkdownVariant: true })
- ).toBe('pass')
+ expect(negotiateMarkdown({ acceptHeader: '*/*' }, { hasMarkdownVariant: true })).toBe('pass')
})
})
describe('q-value negotiation', () => {
it('serves HTML for browser-style Accept', () => {
expect(
- negotiateMarkdown(
- { acceptHeader: BROWSER_ACCEPT, userAgent: '' },
- { hasMarkdownVariant: true }
- )
+ negotiateMarkdown({ acceptHeader: BROWSER_ACCEPT }, { hasMarkdownVariant: true })
).toBe('pass')
})
it('serves markdown when explicitly requested', () => {
expect(
- negotiateMarkdown(
- { acceptHeader: 'text/markdown', userAgent: '' },
- { hasMarkdownVariant: true }
- )
+ negotiateMarkdown({ acceptHeader: 'text/markdown' }, { hasMarkdownVariant: true })
).toBe('markdown')
})
it('serves markdown when its q-value beats html', () => {
expect(
negotiateMarkdown(
- { acceptHeader: 'text/html;q=0.5, text/markdown;q=1.0', userAgent: '' },
+ { acceptHeader: 'text/html;q=0.5, text/markdown;q=1.0' },
{ hasMarkdownVariant: true }
)
).toBe('markdown')
@@ -122,7 +82,7 @@ describe('negotiateMarkdown', () => {
it('serves HTML when its q-value beats markdown', () => {
expect(
negotiateMarkdown(
- { acceptHeader: 'text/html;q=1.0, text/markdown;q=0.5', userAgent: '' },
+ { acceptHeader: 'text/html;q=1.0, text/markdown;q=0.5' },
{ hasMarkdownVariant: true }
)
).toBe('pass')
@@ -131,7 +91,7 @@ describe('negotiateMarkdown', () => {
it('breaks an explicit md/html tie toward markdown', () => {
expect(
negotiateMarkdown(
- { acceptHeader: 'text/markdown, text/html, */*', userAgent: '' },
+ { acceptHeader: 'text/markdown, text/html, */*' },
{ hasMarkdownVariant: true }
)
).toBe('markdown')
@@ -140,7 +100,7 @@ describe('negotiateMarkdown', () => {
it('does not serve markdown when the client rejects it (q=0)', () => {
expect(
negotiateMarkdown(
- { acceptHeader: 'text/markdown;q=0, text/html;q=1.0', userAgent: '' },
+ { acceptHeader: 'text/markdown;q=0, text/html;q=1.0' },
{ hasMarkdownVariant: true }
)
).toBe('pass')
@@ -149,7 +109,7 @@ describe('negotiateMarkdown', () => {
it('tolerates OWS around the q parameter (RFC 9110)', () => {
expect(
negotiateMarkdown(
- { acceptHeader: 'text/html ; q = 1.0, text/markdown ; q = 0.5', userAgent: '' },
+ { acceptHeader: 'text/html ; q = 1.0, text/markdown ; q = 0.5' },
{ hasMarkdownVariant: true }
)
).toBe('pass')
@@ -158,7 +118,7 @@ describe('negotiateMarkdown', () => {
it('ignores out-of-range q-values (falls back to 1.0; tie -> markdown)', () => {
expect(
negotiateMarkdown(
- { acceptHeader: 'text/html;q=2.0, text/markdown;q=1.0', userAgent: '' },
+ { acceptHeader: 'text/html;q=2.0, text/markdown;q=1.0' },
{ hasMarkdownVariant: true }
)
).toBe('markdown')
diff --git a/packages/common/markdown-negotiation.ts b/packages/common/markdown-negotiation.ts
index 28f8416815a..c9373d282e7 100644
--- a/packages/common/markdown-negotiation.ts
+++ b/packages/common/markdown-negotiation.ts
@@ -1,17 +1,9 @@
-// Live-fetch agents only. Training crawlers (GPTBot, ClaudeBot, CCBot) are
-// governed by robots.txt; serving them content that differs from the HTML
-// page risks SEO and cloaking penalties.
-const LLM_USER_AGENT = /\bClaude-User\b|\bClaude-Web\b|\bChatGPT-User\b|\bPerplexityBot\b/i
-
// Media ranges (RFC 9110 §5.3.2) ordered most to least specific.
const RANGES = ['text/markdown', 'text/html', 'text/*', '*/*'] as const
type Range = (typeof RANGES)[number]
const Q_PARAM = /^\s*q\s*=\s*([\d.]+)\s*$/i
-// Cap UA length before the regex test to bound CPU on the edge hot path.
-const MAX_UA_LENGTH = 512
-
function isRange(s: string): s is Range {
return (RANGES as readonly string[]).includes(s)
}
@@ -54,12 +46,17 @@ export type MarkdownDecision = 'markdown' | 'not-acceptable' | 'pass'
/**
* Content negotiation for routes that can serve either HTML or markdown.
*
- * `hasMarkdownVariant` is false for paths with no markdown representation (they
- * never negotiate). `isMarkdownSuffix` forces markdown for an explicit `.md`
- * request; callers that handle `.md` upstream can leave it false.
+ * Markdown is served only on an explicit client signal: a `.md` request
+ * (`isMarkdownSuffix`) or an Accept header preferring text/markdown. There is
+ * deliberately no user-agent detection: responses that vary by UA poison
+ * UA-blind CDN caches, and at least one major agent's reader hard-fails on
+ * markdown it did not explicitly ask for.
+ *
+ * `hasMarkdownVariant` is false for paths with no markdown representation
+ * (they never negotiate).
*/
export function negotiateMarkdown(
- { acceptHeader, userAgent }: { acceptHeader: string; userAgent: string },
+ { acceptHeader }: { acceptHeader: string },
{
hasMarkdownVariant,
isMarkdownSuffix = false,
@@ -67,18 +64,14 @@ export function negotiateMarkdown(
): MarkdownDecision {
if (!hasMarkdownVariant) return 'pass'
- // LLM agents and an explicit `.md` request always get markdown.
- if (LLM_USER_AGENT.test(userAgent.slice(0, MAX_UA_LENGTH)) || isMarkdownSuffix) {
- return 'markdown'
- }
+ if (isMarkdownSuffix) return 'markdown'
// No Accept header = browser/default client: serve HTML, never 406.
if (!acceptHeader) return 'pass'
const accept = parseAccept(acceptHeader)
- // 406 when Accept rejects every type this route can produce. Only reached for
- // non-LLM, non-`.md` clients that sent an Accept header (guards above), so a
+ // 406 when Accept rejects every type this route can produce, so a
// deliberate `Accept: application/json` gets a clean 406 instead of HTML.
if (accept.markdown === 0 && accept.html === 0) return 'not-acceptable'