The agent that turns websites into APIs!
feat(website): SEO + agent discovery metadata
kalil032155bed8c35471
diff --git a/website/content/docs/index.mdx b/website/content/docs/index.mdxindex f3dce6a..5e07dde 100644--- a/website/content/docs/index.mdx+++ b/website/content/docs/index.mdx@@ -1,6 +1,6 @@---title: Introduction-description: Reverse-engineer any website's API and get a production-ready Python client in minutes.+description: Capture browser traffic and turn it into a production-ready Python API client.---**Reverse API Engineer** is a CLI tool that captures browser traffic and usesdiff --git a/website/public/_headers b/website/public/_headersnew file mode 100644index 0000000..374431c--- /dev/null+++ b/website/public/_headers@@ -0,0 +1,11 @@+# RFC 8288 Link response headers for agent discovery.+# - service-doc → human-readable docs (IANA registered)+# - describedby → /llms.txt machine-readable site summary (IANA registered)+# Applied per-path via Cloudflare Pages' _headers file.++/+ Link: </docs>; rel="service-doc"+ Link: </llms.txt>; rel="describedby"++/docs/*+ Link: </llms.txt>; rel="describedby"diff --git a/website/src/app/(home)/page.tsx b/website/src/app/(home)/page.tsxindex ae58b75..01cce56 100644--- a/website/src/app/(home)/page.tsx+++ b/website/src/app/(home)/page.tsx@@ -1,22 +1,23 @@import Link from 'next/link';import type { Metadata } from 'next';import { ArrowRightIcon } from 'lucide-react';-import { githubUrl } from '@/lib/shared';+import { appName, appTagline, gitConfig, githubUrl, pypiUrl, siteUrl } from '@/lib/shared';import { HeroCyclingPhrase } from '@/components/hero-cycling-phrase';import { InstallCommand } from '@/components/install-command';import { BuiltInTheOpen } from '@/components/built-in-the-open';+import { JsonLd } from '@/components/json-ld';export const metadata: Metadata = {- title: 'Generate API Clients from Browser Traffic',+ title: 'Turn websites into APIs',description:- 'Capture browser traffic with Playwright or Chrome DevTools MCP and generate typed Python, JavaScript, or TypeScript API clients.',+ 'Capture browser traffic and generate typed Python, JavaScript, or TypeScript API clients.',alternates: {canonical: '/',},openGraph: {- title: 'Generate API Clients from Browser Traffic',+ title: 'Turn websites into APIs',description:- 'Capture browser traffic and turn HAR recordings into typed API clients.',+ 'Capture browser traffic and generate typed Python, JavaScript, or TypeScript API clients.',url: '/',},};@@ -29,9 +30,34 @@ function GithubIcon({ className }: { className?: string }) {);}+const softwareJsonLd = {+ '@context': 'https://schema.org',+ '@type': 'SoftwareApplication',+ name: appName,+ description: appTagline,+ applicationCategory: 'DeveloperApplication',+ operatingSystem: 'macOS, Linux, Windows',+ url: siteUrl,+ downloadUrl: pypiUrl,+ codeRepository: githubUrl,+ license: 'https://opensource.org/licenses/MIT',+ programmingLanguage: ['Python', 'JavaScript', 'TypeScript'],+ offers: {+ '@type': 'Offer',+ price: '0',+ priceCurrency: 'USD',+ },+ author: {+ '@type': 'Person',+ name: gitConfig.user,+ url: `https://github.com/${gitConfig.user}`,+ },+};+export default function HomePage() {return (<main className="flex-1">+ <JsonLd data={softwareJsonLd} /><div className="flex flex-col h-[calc(100svh-3.5rem)] min-h-[640px]"><Hero /><Marquee />diff --git a/website/src/app/banner/page.tsx b/website/src/app/banner/page.tsxindex 824cb93..56346e4 100644--- a/website/src/app/banner/page.tsx+++ b/website/src/app/banner/page.tsx@@ -14,6 +14,31 @@ export default function BannerPage() {<divclassName="relative w-full h-full bg-color-mesh overflow-hidden flex items-center justify-center">+ {/* Logo — top-left corner */}+ <div className="absolute top-10 left-12 flex items-baseline gap-2">+ <span+ className="font-display select-none leading-none inline-block italic"+ style={{+ fontSize: 56,+ color: 'var(--color-fd-primary)',+ fontVariationSettings: "'opsz' 144, 'SOFT' 100, 'WONK' 1",+ }}+ aria-hidden="true"+ >+ *+ </span>+ <span+ className="font-display italic text-ink leading-none"+ style={{+ fontSize: 64,+ letterSpacing: '-0.05em',+ fontVariationSettings: "'opsz' 144, 'SOFT' 100, 'WONK' 1",+ }}+ >+ rae+ </span>+ </div>+{/* Ambient asterisks */}<spanaria-hidden="true"@@ -30,55 +55,40 @@ export default function BannerPage() {*</span>- {/* Headline + logo, vertically stacked, centered */}- <div className="relative flex flex-col items-center text-center gap-12">- <h1- className="font-display text-ink leading-[0.95]"+ {/* Headline — centered */}+ <h1+ className="relative font-display text-ink leading-[0.95] text-center"+ style={{+ fontSize: 104,+ fontVariationSettings: "'opsz' 144, 'SOFT' 30, 'WONK' 1",+ fontWeight: 500,+ letterSpacing: '-0.045em',+ }}+ >+ Turn websites<br />+ <em+ className="italic relative inline-block text-ink"style={{- fontSize: 96,- fontVariationSettings: "'opsz' 144, 'SOFT' 30, 'WONK' 1",- fontWeight: 500,- letterSpacing: '-0.045em',+ fontVariationSettings: "'opsz' 144, 'SOFT' 100, 'WONK' 1",+ fontWeight: 400,+ paddingBottom: '0.08em',}}>- Turn any website<br />- <em- className="italic"- style={{- fontVariationSettings: "'opsz' 144, 'SOFT' 100, 'WONK' 1",- fontWeight: 400,- color: 'var(--color-fd-primary)',- }}- >- into an API.- </em>- </h1>-- {/* rae logo — large, centered under the headline */}- <div className="flex items-baseline gap-3">+ into APIs.<span- className="font-display select-none leading-none inline-block italic"- style={{- fontSize: 88,- color: 'var(--color-fd-primary)',- fontVariationSettings: "'opsz' 144, 'SOFT' 100, 'WONK' 1",- }}aria-hidden="true"- >- *- </span>- <span- className="font-display italic text-ink leading-none"style={{- fontSize: 96,- letterSpacing: '-0.05em',- fontVariationSettings: "'opsz' 144, 'SOFT' 100, 'WONK' 1",+ position: 'absolute',+ left: 0,+ right: 0,+ bottom: '-0.02em',+ height: 6,+ background: 'var(--color-fd-primary)',+ borderRadius: 2,}}- >- rae- </span>- </div>- </div>+ />+ </em>+ </h1></div>{/* Tiny caption under the frame — tells the user what to do */}diff --git a/website/src/app/docs/[[...slug]]/page.tsx b/website/src/app/docs/[[...slug]]/page.tsxindex 0c16575..6a1ca14 100644--- a/website/src/app/docs/[[...slug]]/page.tsx+++ b/website/src/app/docs/[[...slug]]/page.tsx@@ -13,7 +13,8 @@ import {flattenTree,} from '@/lib/docs';import { getDocsMdxComponents } from '@/components/docs/mdx-components';-import { appName } from '@/lib/shared';+import { JsonLd } from '@/components/json-ld';+import { appName, siteUrl } from '@/lib/shared';interface PageProps {params: Promise<{ slug?: string[] }>;@@ -48,8 +49,32 @@ export default async function Page({ params }: PageProps) {const idx = flat.findIndex((p) => p.url === page.url);const prev = idx > 0 ? flat[idx - 1] : null;const next = idx >= 0 && idx < flat.length - 1 ? flat[idx + 1] : null;++ const breadcrumbItems = [+ { name: 'Docs', url: new URL('/docs', siteUrl).toString() },+ ...page.slug.map((seg, i) => {+ const segUrl = `/docs/${page.slug.slice(0, i + 1).join('/')}`;+ const node = flat.find((p) => p.url === segUrl);+ return {+ name: node?.title ?? seg,+ url: new URL(segUrl, siteUrl).toString(),+ };+ }),+ ];+ const breadcrumbJsonLd = {+ '@context': 'https://schema.org',+ '@type': 'BreadcrumbList',+ itemListElement: breadcrumbItems.map((it, i) => ({+ '@type': 'ListItem',+ position: i + 1,+ name: it.name,+ item: it.url,+ })),+ };+return (- <article className="max-w-[740px] flex flex-col">+ <article className="max-w-[740px] mx-auto flex flex-col">+ {page.slug.length > 0 ? <JsonLd data={breadcrumbJsonLd} /> : null}{/* Header — title + description, tight rhythm before the body */}<header className="pb-4"><h1 className="font-display text-4xl md:text-5xl tracking-[-0.04em] text-ink leading-[1.05]">diff --git a/website/src/app/docs/layout.tsx b/website/src/app/docs/layout.tsxindex 1126a5c..445ea27 100644--- a/website/src/app/docs/layout.tsx+++ b/website/src/app/docs/layout.tsx@@ -1,3 +1,4 @@+import { ChevronDownIcon } from 'lucide-react';import { getDocTree } from '@/lib/docs';import { SiteNav } from '@/components/site-nav';import { DocsSidebar } from '@/components/docs/sidebar';@@ -9,27 +10,40 @@ export default function Layout({ children }: { children: React.ReactNode }) {<><SiteNav />- {/* Fixed sidebar — true viewport vertical centring, left edge- aligned with the docs container's content edge. Stays fully- visible while scrolling. */}+ {/* lg+ : fixed sidebar, vertically centred against the viewport, left+ edge aligned with the docs container. < lg: hidden via CSS — the+ mobile <details> menu below takes over. */}<aside className="docs-sidebar-fixed">- <nav- className="w-full max-h-full overflow-y-auto pr-3"+ <div+ className="h-full overflow-y-auto"style={{ pointerEvents: 'auto' }}>- <DocsSidebar tree={tree} />- </nav>+ {/* `safe center` falls back to `flex-start` when the nav overflows,+ so the scrollbar stays usable instead of pushing content off the+ top of the scroll container. */}+ <nav className="min-h-full flex flex-col [justify-content:safe_center] px-3 py-2">+ <DocsSidebar tree={tree} />+ </nav>+ </div></aside><div className="mx-auto max-w-7xl px-6 lg:px-10 w-full">- <div- className="grid gap-12 lg:gap-16"- style={{ gridTemplateColumns: '240px minmax(0, 1fr)' }}- >- {/* Placeholder — preserves the main content's horizontal position- now that the sidebar is fixed (out of flow). */}- <div aria-hidden />- <main className="min-w-0 pt-6 pb-16 md:pt-8">{children}</main>+ {/* Mobile docs menu — collapsible, no JS needed. Hidden on lg+. */}+ <details className="lg:hidden mt-4 mx-auto max-w-[740px] group rounded-xl border border-ink/10">+ <summary className="flex items-center justify-between px-4 py-3 cursor-pointer list-none font-mono text-xs uppercase tracking-widest text-ink-soft">+ <span>Documentation</span>+ <ChevronDownIcon className="size-4 transition-transform group-open:rotate-180" />+ </summary>+ <div className="px-3 pb-3 pt-1 border-t border-ink/10 max-h-[60vh] overflow-y-auto">+ <DocsSidebar tree={tree} />+ </div>+ </details>++ <div className="lg:grid lg:gap-8 xl:gap-0 lg:[grid-template-columns:240px_minmax(0,1fr)]">+ {/* Placeholder — preserves main's horizontal position next to the+ fixed sidebar on lg+. Hidden under lg. */}+ <div aria-hidden className="hidden lg:block" />+ <main className="min-w-0 pt-6 pb-4 md:pt-8">{children}</main></div></div></>diff --git a/website/src/app/global.css b/website/src/app/global.cssindex 798629a..f56f1a7 100644--- a/website/src/app/global.css+++ b/website/src/app/global.css@@ -388,21 +388,23 @@ body {.docs-sidebar-fixed {position: fixed;- top: 50%;- transform: translateY(-50%);+ /* SiteNav is 3.5rem; leave 1.5rem of breathing room above + 1.5rem+ below the viewport. */+ top: 5rem;+ bottom: 1.5rem;width: 240px;- max-height: calc(100vh - 6rem);- /* 40rem = half of max-w-7xl (80rem). 1.5rem = px-6 default. */- left: max(1.5rem, calc(50vw - 40rem + 1.5rem));z-index: 10;- display: flex;- align-items: center;pointer-events: none;+ /* Hidden below lg; the mobile <details> menu takes over. */+ display: none;}-@media (min-width: 1024px) {+@media (min-width: 64rem) {+ /* 64rem matches Tailwind v4's lg: breakpoint so this stays in lockstep+ with `lg:hidden` / `lg:grid` / `lg:block` regardless of root font size. */.docs-sidebar-fixed {- /* lg: px-10 = 2.5rem */+ display: block;+ /* lg: container px-10 = 2.5rem. 40rem = half of max-w-7xl (80rem). */left: max(2.5rem, calc(50vw - 40rem + 2.5rem));}}diff --git a/website/src/app/layout.tsx b/website/src/app/layout.tsxindex 8ed1c5d..e359e36 100644--- a/website/src/app/layout.tsx+++ b/website/src/app/layout.tsx@@ -1,14 +1,14 @@import './global.css';import { Inter, JetBrains_Mono, Fraunces } from 'next/font/google';import type { Metadata } from 'next';-import { appName, appTagline, githubUrl, siteUrl } from '@/lib/shared';+import { appName, appTagline, gitConfig, siteUrl } from '@/lib/shared';import { Providers } from '@/components/providers';export const metadata: Metadata = {metadataBase: new URL(siteUrl),applicationName: appName,title: {- default: `${appName} · Generate API Clients from Browser Traffic`,+ default: `${appName} · Turn websites into APIs`,template: `%s · ${appName}`,},description: appTagline,@@ -25,23 +25,23 @@ export const metadata: Metadata = {openGraph: {type: 'website',siteName: appName,- title: `${appName} · Generate API Clients from Browser Traffic`,+ title: `${appName} · Turn websites into APIs`,description: appTagline,url: '/',images: [{- url: `${githubUrl}/raw/main/assets/reverse-api-banner.jpg`,- width: 1200,- height: 630,+ url: `https://raw.githubusercontent.com/${gitConfig.user}/${gitConfig.repo}/main/assets/reverse-api-banner.jpg`,+ width: 2564,+ height: 1284,alt: `${appName} banner`,},],},twitter: {card: 'summary_large_image',- title: `${appName} · Generate API Clients from Browser Traffic`,+ title: `${appName} · Turn websites into APIs`,description: appTagline,- images: [`${githubUrl}/raw/main/assets/reverse-api-banner.jpg`],+ images: [`https://raw.githubusercontent.com/${gitConfig.user}/${gitConfig.repo}/main/assets/reverse-api-banner.jpg`],},robots: {index: true,diff --git a/website/src/app/llms-full.txt/route.ts b/website/src/app/llms-full.txt/route.tsnew file mode 100644index 0000000..c9b05c4--- /dev/null+++ b/website/src/app/llms-full.txt/route.ts@@ -0,0 +1,36 @@+import { NextResponse } from 'next/server';+import { getAllDocPages } from '@/lib/docs';+import { appName, appTagline, siteUrl, githubUrl, pypiUrl } from '@/lib/shared';++export const dynamic = 'force-static';++export function GET() {+ const pages = getAllDocPages();++ const sections = pages+ .map((p) => {+ const url = new URL(p.url, siteUrl).toString();+ const title = p.frontmatter.title;+ const description = p.frontmatter.description+ ? `\n> ${p.frontmatter.description}\n`+ : '';+ return `\n\n# ${title}\n\nSource: ${url}${description}\n${p.content.trim()}`;+ })+ .join('\n\n---');++ const header = `# ${appName}++> ${appTagline}++Repository: ${githubUrl}+PyPI: ${pypiUrl}+License: MIT++This file concatenates every documentation page in raw MDX. Use it to give+an LLM full context on the project in a single prompt.+`;++ return new NextResponse(header + sections, {+ headers: { 'Content-Type': 'text/plain; charset=utf-8' },+ });+}diff --git a/website/src/app/llms.txt/route.ts b/website/src/app/llms.txt/route.tsnew file mode 100644index 0000000..985c23e--- /dev/null+++ b/website/src/app/llms.txt/route.ts@@ -0,0 +1,34 @@+import { NextResponse } from 'next/server';+import { getDocTree, flattenTree } from '@/lib/docs';+import { appName, appTagline, siteUrl, githubUrl, pypiUrl } from '@/lib/shared';++export const dynamic = 'force-static';++export function GET() {+ const pages = flattenTree(getDocTree());++ const docLines = pages+ .map((p) => `- [${p.title}](${new URL(p.url, siteUrl).toString()})`)+ .join('\n');++ const body = `# ${appName}++> ${appTagline}++${appName} is an open-source CLI that captures browser traffic with Playwright (or Chrome DevTools MCP) and uses your configured AI SDK to generate a typed Python, JavaScript, or TypeScript API client from the recorded HAR.++## Documentation++${docLines}++## Project++- Repository: ${githubUrl}+- PyPI: ${pypiUrl}+- License: MIT+`;++ return new NextResponse(body, {+ headers: { 'Content-Type': 'text/plain; charset=utf-8' },+ });+}diff --git a/website/src/app/robots.ts b/website/src/app/robots.tsdeleted file mode 100644index 054f657..0000000--- a/website/src/app/robots.ts+++ /dev/null@@ -1,14 +0,0 @@-import type { MetadataRoute } from 'next';-import { siteUrl } from '@/lib/shared';--export const dynamic = 'force-static';--export default function robots(): MetadataRoute.Robots {- return {- rules: {- userAgent: '*',- allow: '/',- },- sitemap: new URL('/sitemap.xml', siteUrl).toString(),- };-}diff --git a/website/src/app/robots.txt/route.ts b/website/src/app/robots.txt/route.tsnew file mode 100644index 0000000..033ec21--- /dev/null+++ b/website/src/app/robots.txt/route.ts@@ -0,0 +1,22 @@+import { NextResponse } from 'next/server';+import { siteUrl } from '@/lib/shared';++export const dynamic = 'force-static';++export function GET() {+ const sitemap = new URL('/sitemap.xml', siteUrl).toString();++ // Content-Signal: AI-preferences draft (contentsignals.org).+ // rae is MIT-licensed open source — we want to be trained on, indexed,+ // and cited by AI tools.+ const body = `User-agent: *+Allow: /+Content-Signal: ai-train=yes, search=yes, ai-input=yes++Sitemap: ${sitemap}+`;++ return new NextResponse(body, {+ headers: { 'Content-Type': 'text/plain; charset=utf-8' },+ });+}diff --git a/website/src/components/docs/mdx/tabs.tsx b/website/src/components/docs/mdx/tabs.tsxindex 158113b..9d84417 100644--- a/website/src/components/docs/mdx/tabs.tsx+++ b/website/src/components/docs/mdx/tabs.tsx@@ -39,7 +39,7 @@ export function Tabs({ items, children }: TabsProps) {<div className="mt-5"><divrole="tablist"- className="flex items-center gap-5 border-b border-ink/10 overflow-x-auto"+ className="flex items-center gap-5 border-b border-ink/10 overflow-x-auto overflow-y-hidden">{tabs.map((t) => {const isActive = t.value === active;diff --git a/website/src/components/json-ld.tsx b/website/src/components/json-ld.tsxnew file mode 100644index 0000000..266c131--- /dev/null+++ b/website/src/components/json-ld.tsx@@ -0,0 +1,16 @@+/* Schema.org JSON-LD block for SEO + LLM crawler signals.+ Must render as a plain <script> tag in the static HTML (not via+ next/script, which only injects through React hydration and so is+ invisible to non-JS crawlers). The `<` escape prevents a `</script>`+ sequence from inside a string value breaking out of the tag. */++export function JsonLd({ data }: { data: unknown }) {+ const json = JSON.stringify(data).replace(/</g, '\\u003c');+ return (+ <script+ type="application/ld+json"+ // eslint-disable-next-line react/no-danger+ dangerouslySetInnerHTML={{ __html: json }}+ />+ );+}diff --git a/website/src/lib/shared.ts b/website/src/lib/shared.tsindex e25bc51..0ac1edb 100644--- a/website/src/lib/shared.ts+++ b/website/src/lib/shared.ts@@ -1,5 +1,5 @@export const appName = 'Reverse API Engineer';-export const appTagline = 'Generate typed API clients from browser traffic.';+export const appTagline = 'Turn websites into APIs.';export const siteUrl = process.env.NEXT_PUBLIC_SITE_URL ?? 'https://reverseapi.dev';export const docsRoute = '/docs';export const docsImageRoute = '/og/docs';