Skip to content

The agent that turns websites into APIs!

feat(website): SEO + agent discovery metadata

kalil032155bed8c35471
diff --git a/website/content/docs/index.mdx b/website/content/docs/index.mdx
index f3dce6a..5e07dde 100644
--- a/website/content/docs/index.mdx
+++ b/website/content/docs/index.mdx
@@ -1,6 +1,6 @@
---
title: Introduction
-description: Reverse-engineer any website's API and get a production-ready Python client in minutes.
+description: Capture browser traffic and turn it into a production-ready Python API client.
---
**Reverse API Engineer** is a CLI tool that captures browser traffic and uses
diff --git a/website/public/_headers b/website/public/_headers
new file mode 100644
index 0000000..374431c
--- /dev/null
+++ b/website/public/_headers
@@ -0,0 +1,11 @@
+# RFC 8288 Link response headers for agent discovery.
+# - service-doc → human-readable docs (IANA registered)
+# - describedby → /llms.txt machine-readable site summary (IANA registered)
+# Applied per-path via Cloudflare Pages' _headers file.
+
+/
+ Link: </docs>; rel="service-doc"
+ Link: </llms.txt>; rel="describedby"
+
+/docs/*
+ Link: </llms.txt>; rel="describedby"
diff --git a/website/src/app/(home)/page.tsx b/website/src/app/(home)/page.tsx
index ae58b75..01cce56 100644
--- a/website/src/app/(home)/page.tsx
+++ b/website/src/app/(home)/page.tsx
@@ -1,22 +1,23 @@
import Link from 'next/link';
import type { Metadata } from 'next';
import { ArrowRightIcon } from 'lucide-react';
-import { githubUrl } from '@/lib/shared';
+import { appName, appTagline, gitConfig, githubUrl, pypiUrl, siteUrl } from '@/lib/shared';
import { HeroCyclingPhrase } from '@/components/hero-cycling-phrase';
import { InstallCommand } from '@/components/install-command';
import { BuiltInTheOpen } from '@/components/built-in-the-open';
+import { JsonLd } from '@/components/json-ld';
export const metadata: Metadata = {
- title: 'Generate API Clients from Browser Traffic',
+ title: 'Turn websites into APIs',
description:
- 'Capture browser traffic with Playwright or Chrome DevTools MCP and generate typed Python, JavaScript, or TypeScript API clients.',
+ 'Capture browser traffic and generate typed Python, JavaScript, or TypeScript API clients.',
alternates: {
canonical: '/',
},
openGraph: {
- title: 'Generate API Clients from Browser Traffic',
+ title: 'Turn websites into APIs',
description:
- 'Capture browser traffic and turn HAR recordings into typed API clients.',
+ 'Capture browser traffic and generate typed Python, JavaScript, or TypeScript API clients.',
url: '/',
},
};
@@ -29,9 +30,34 @@ function GithubIcon({ className }: { className?: string }) {
);
}
+const softwareJsonLd = {
+ '@context': 'https://schema.org',
+ '@type': 'SoftwareApplication',
+ name: appName,
+ description: appTagline,
+ applicationCategory: 'DeveloperApplication',
+ operatingSystem: 'macOS, Linux, Windows',
+ url: siteUrl,
+ downloadUrl: pypiUrl,
+ codeRepository: githubUrl,
+ license: 'https://opensource.org/licenses/MIT',
+ programmingLanguage: ['Python', 'JavaScript', 'TypeScript'],
+ offers: {
+ '@type': 'Offer',
+ price: '0',
+ priceCurrency: 'USD',
+ },
+ author: {
+ '@type': 'Person',
+ name: gitConfig.user,
+ url: `https://github.com/${gitConfig.user}`,
+ },
+};
+
export default function HomePage() {
return (
<main className="flex-1">
+ <JsonLd data={softwareJsonLd} />
<div className="flex flex-col h-[calc(100svh-3.5rem)] min-h-[640px]">
<Hero />
<Marquee />
diff --git a/website/src/app/banner/page.tsx b/website/src/app/banner/page.tsx
index 824cb93..56346e4 100644
--- a/website/src/app/banner/page.tsx
+++ b/website/src/app/banner/page.tsx
@@ -14,6 +14,31 @@ export default function BannerPage() {
<div
className="relative w-full h-full bg-color-mesh overflow-hidden flex items-center justify-center"
>
+ {/* Logo — top-left corner */}
+ <div className="absolute top-10 left-12 flex items-baseline gap-2">
+ <span
+ className="font-display select-none leading-none inline-block italic"
+ style={{
+ fontSize: 56,
+ color: 'var(--color-fd-primary)',
+ fontVariationSettings: "'opsz' 144, 'SOFT' 100, 'WONK' 1",
+ }}
+ aria-hidden="true"
+ >
+ *
+ </span>
+ <span
+ className="font-display italic text-ink leading-none"
+ style={{
+ fontSize: 64,
+ letterSpacing: '-0.05em',
+ fontVariationSettings: "'opsz' 144, 'SOFT' 100, 'WONK' 1",
+ }}
+ >
+ rae
+ </span>
+ </div>
+
{/* Ambient asterisks */}
<span
aria-hidden="true"
@@ -30,55 +55,40 @@ export default function BannerPage() {
*
</span>
- {/* Headline + logo, vertically stacked, centered */}
- <div className="relative flex flex-col items-center text-center gap-12">
- <h1
- className="font-display text-ink leading-[0.95]"
+ {/* Headline — centered */}
+ <h1
+ className="relative font-display text-ink leading-[0.95] text-center"
+ style={{
+ fontSize: 104,
+ fontVariationSettings: "'opsz' 144, 'SOFT' 30, 'WONK' 1",
+ fontWeight: 500,
+ letterSpacing: '-0.045em',
+ }}
+ >
+ Turn websites<br />
+ <em
+ className="italic relative inline-block text-ink"
style={{
- fontSize: 96,
- fontVariationSettings: "'opsz' 144, 'SOFT' 30, 'WONK' 1",
- fontWeight: 500,
- letterSpacing: '-0.045em',
+ fontVariationSettings: "'opsz' 144, 'SOFT' 100, 'WONK' 1",
+ fontWeight: 400,
+ paddingBottom: '0.08em',
}}
>
- Turn any website<br />
- <em
- className="italic"
- style={{
- fontVariationSettings: "'opsz' 144, 'SOFT' 100, 'WONK' 1",
- fontWeight: 400,
- color: 'var(--color-fd-primary)',
- }}
- >
- into an API.
- </em>
- </h1>
-
- {/* rae logo — large, centered under the headline */}
- <div className="flex items-baseline gap-3">
+ into APIs.
<span
- className="font-display select-none leading-none inline-block italic"
- style={{
- fontSize: 88,
- color: 'var(--color-fd-primary)',
- fontVariationSettings: "'opsz' 144, 'SOFT' 100, 'WONK' 1",
- }}
aria-hidden="true"
- >
- *
- </span>
- <span
- className="font-display italic text-ink leading-none"
style={{
- fontSize: 96,
- letterSpacing: '-0.05em',
- fontVariationSettings: "'opsz' 144, 'SOFT' 100, 'WONK' 1",
+ position: 'absolute',
+ left: 0,
+ right: 0,
+ bottom: '-0.02em',
+ height: 6,
+ background: 'var(--color-fd-primary)',
+ borderRadius: 2,
}}
- >
- rae
- </span>
- </div>
- </div>
+ />
+ </em>
+ </h1>
</div>
{/* Tiny caption under the frame — tells the user what to do */}
diff --git a/website/src/app/docs/[[...slug]]/page.tsx b/website/src/app/docs/[[...slug]]/page.tsx
index 0c16575..6a1ca14 100644
--- a/website/src/app/docs/[[...slug]]/page.tsx
+++ b/website/src/app/docs/[[...slug]]/page.tsx
@@ -13,7 +13,8 @@ import {
flattenTree,
} from '@/lib/docs';
import { getDocsMdxComponents } from '@/components/docs/mdx-components';
-import { appName } from '@/lib/shared';
+import { JsonLd } from '@/components/json-ld';
+import { appName, siteUrl } from '@/lib/shared';
interface PageProps {
params: Promise<{ slug?: string[] }>;
@@ -48,8 +49,32 @@ export default async function Page({ params }: PageProps) {
const idx = flat.findIndex((p) => p.url === page.url);
const prev = idx > 0 ? flat[idx - 1] : null;
const next = idx >= 0 && idx < flat.length - 1 ? flat[idx + 1] : null;
+
+ const breadcrumbItems = [
+ { name: 'Docs', url: new URL('/docs', siteUrl).toString() },
+ ...page.slug.map((seg, i) => {
+ const segUrl = `/docs/${page.slug.slice(0, i + 1).join('/')}`;
+ const node = flat.find((p) => p.url === segUrl);
+ return {
+ name: node?.title ?? seg,
+ url: new URL(segUrl, siteUrl).toString(),
+ };
+ }),
+ ];
+ const breadcrumbJsonLd = {
+ '@context': 'https://schema.org',
+ '@type': 'BreadcrumbList',
+ itemListElement: breadcrumbItems.map((it, i) => ({
+ '@type': 'ListItem',
+ position: i + 1,
+ name: it.name,
+ item: it.url,
+ })),
+ };
+
return (
- <article className="max-w-[740px] flex flex-col">
+ <article className="max-w-[740px] mx-auto flex flex-col">
+ {page.slug.length > 0 ? <JsonLd data={breadcrumbJsonLd} /> : null}
{/* Header — title + description, tight rhythm before the body */}
<header className="pb-4">
<h1 className="font-display text-4xl md:text-5xl tracking-[-0.04em] text-ink leading-[1.05]">
diff --git a/website/src/app/docs/layout.tsx b/website/src/app/docs/layout.tsx
index 1126a5c..445ea27 100644
--- a/website/src/app/docs/layout.tsx
+++ b/website/src/app/docs/layout.tsx
@@ -1,3 +1,4 @@
+import { ChevronDownIcon } from 'lucide-react';
import { getDocTree } from '@/lib/docs';
import { SiteNav } from '@/components/site-nav';
import { DocsSidebar } from '@/components/docs/sidebar';
@@ -9,27 +10,40 @@ export default function Layout({ children }: { children: React.ReactNode }) {
<>
<SiteNav />
- {/* Fixed sidebar — true viewport vertical centring, left edge
- aligned with the docs container's content edge. Stays fully
- visible while scrolling. */}
+ {/* lg+ : fixed sidebar, vertically centred against the viewport, left
+ edge aligned with the docs container. < lg: hidden via CSS — the
+ mobile <details> menu below takes over. */}
<aside className="docs-sidebar-fixed">
- <nav
- className="w-full max-h-full overflow-y-auto pr-3"
+ <div
+ className="h-full overflow-y-auto"
style={{ pointerEvents: 'auto' }}
>
- <DocsSidebar tree={tree} />
- </nav>
+ {/* `safe center` falls back to `flex-start` when the nav overflows,
+ so the scrollbar stays usable instead of pushing content off the
+ top of the scroll container. */}
+ <nav className="min-h-full flex flex-col [justify-content:safe_center] px-3 py-2">
+ <DocsSidebar tree={tree} />
+ </nav>
+ </div>
</aside>
<div className="mx-auto max-w-7xl px-6 lg:px-10 w-full">
- <div
- className="grid gap-12 lg:gap-16"
- style={{ gridTemplateColumns: '240px minmax(0, 1fr)' }}
- >
- {/* Placeholder — preserves the main content's horizontal position
- now that the sidebar is fixed (out of flow). */}
- <div aria-hidden />
- <main className="min-w-0 pt-6 pb-16 md:pt-8">{children}</main>
+ {/* Mobile docs menu — collapsible, no JS needed. Hidden on lg+. */}
+ <details className="lg:hidden mt-4 mx-auto max-w-[740px] group rounded-xl border border-ink/10">
+ <summary className="flex items-center justify-between px-4 py-3 cursor-pointer list-none font-mono text-xs uppercase tracking-widest text-ink-soft">
+ <span>Documentation</span>
+ <ChevronDownIcon className="size-4 transition-transform group-open:rotate-180" />
+ </summary>
+ <div className="px-3 pb-3 pt-1 border-t border-ink/10 max-h-[60vh] overflow-y-auto">
+ <DocsSidebar tree={tree} />
+ </div>
+ </details>
+
+ <div className="lg:grid lg:gap-8 xl:gap-0 lg:[grid-template-columns:240px_minmax(0,1fr)]">
+ {/* Placeholder — preserves main's horizontal position next to the
+ fixed sidebar on lg+. Hidden under lg. */}
+ <div aria-hidden className="hidden lg:block" />
+ <main className="min-w-0 pt-6 pb-4 md:pt-8">{children}</main>
</div>
</div>
</>
diff --git a/website/src/app/global.css b/website/src/app/global.css
index 798629a..f56f1a7 100644
--- a/website/src/app/global.css
+++ b/website/src/app/global.css
@@ -388,21 +388,23 @@ body {
.docs-sidebar-fixed {
position: fixed;
- top: 50%;
- transform: translateY(-50%);
+ /* SiteNav is 3.5rem; leave 1.5rem of breathing room above + 1.5rem
+ below the viewport. */
+ top: 5rem;
+ bottom: 1.5rem;
width: 240px;
- max-height: calc(100vh - 6rem);
- /* 40rem = half of max-w-7xl (80rem). 1.5rem = px-6 default. */
- left: max(1.5rem, calc(50vw - 40rem + 1.5rem));
z-index: 10;
- display: flex;
- align-items: center;
pointer-events: none;
+ /* Hidden below lg; the mobile <details> menu takes over. */
+ display: none;
}
-@media (min-width: 1024px) {
+@media (min-width: 64rem) {
+ /* 64rem matches Tailwind v4's lg: breakpoint so this stays in lockstep
+ with `lg:hidden` / `lg:grid` / `lg:block` regardless of root font size. */
.docs-sidebar-fixed {
- /* lg: px-10 = 2.5rem */
+ display: block;
+ /* lg: container px-10 = 2.5rem. 40rem = half of max-w-7xl (80rem). */
left: max(2.5rem, calc(50vw - 40rem + 2.5rem));
}
}
diff --git a/website/src/app/layout.tsx b/website/src/app/layout.tsx
index 8ed1c5d..e359e36 100644
--- a/website/src/app/layout.tsx
+++ b/website/src/app/layout.tsx
@@ -1,14 +1,14 @@
import './global.css';
import { Inter, JetBrains_Mono, Fraunces } from 'next/font/google';
import type { Metadata } from 'next';
-import { appName, appTagline, githubUrl, siteUrl } from '@/lib/shared';
+import { appName, appTagline, gitConfig, siteUrl } from '@/lib/shared';
import { Providers } from '@/components/providers';
export const metadata: Metadata = {
metadataBase: new URL(siteUrl),
applicationName: appName,
title: {
- default: `${appName} · Generate API Clients from Browser Traffic`,
+ default: `${appName} · Turn websites into APIs`,
template: `%s · ${appName}`,
},
description: appTagline,
@@ -25,23 +25,23 @@ export const metadata: Metadata = {
openGraph: {
type: 'website',
siteName: appName,
- title: `${appName} · Generate API Clients from Browser Traffic`,
+ title: `${appName} · Turn websites into APIs`,
description: appTagline,
url: '/',
images: [
{
- url: `${githubUrl}/raw/main/assets/reverse-api-banner.jpg`,
- width: 1200,
- height: 630,
+ url: `https://raw.githubusercontent.com/${gitConfig.user}/${gitConfig.repo}/main/assets/reverse-api-banner.jpg`,
+ width: 2564,
+ height: 1284,
alt: `${appName} banner`,
},
],
},
twitter: {
card: 'summary_large_image',
- title: `${appName} · Generate API Clients from Browser Traffic`,
+ title: `${appName} · Turn websites into APIs`,
description: appTagline,
- images: [`${githubUrl}/raw/main/assets/reverse-api-banner.jpg`],
+ images: [`https://raw.githubusercontent.com/${gitConfig.user}/${gitConfig.repo}/main/assets/reverse-api-banner.jpg`],
},
robots: {
index: true,
diff --git a/website/src/app/llms-full.txt/route.ts b/website/src/app/llms-full.txt/route.ts
new file mode 100644
index 0000000..c9b05c4
--- /dev/null
+++ b/website/src/app/llms-full.txt/route.ts
@@ -0,0 +1,36 @@
+import { NextResponse } from 'next/server';
+import { getAllDocPages } from '@/lib/docs';
+import { appName, appTagline, siteUrl, githubUrl, pypiUrl } from '@/lib/shared';
+
+export const dynamic = 'force-static';
+
+export function GET() {
+ const pages = getAllDocPages();
+
+ const sections = pages
+ .map((p) => {
+ const url = new URL(p.url, siteUrl).toString();
+ const title = p.frontmatter.title;
+ const description = p.frontmatter.description
+ ? `\n> ${p.frontmatter.description}\n`
+ : '';
+ return `\n\n# ${title}\n\nSource: ${url}${description}\n${p.content.trim()}`;
+ })
+ .join('\n\n---');
+
+ const header = `# ${appName}
+
+> ${appTagline}
+
+Repository: ${githubUrl}
+PyPI: ${pypiUrl}
+License: MIT
+
+This file concatenates every documentation page in raw MDX. Use it to give
+an LLM full context on the project in a single prompt.
+`;
+
+ return new NextResponse(header + sections, {
+ headers: { 'Content-Type': 'text/plain; charset=utf-8' },
+ });
+}
diff --git a/website/src/app/llms.txt/route.ts b/website/src/app/llms.txt/route.ts
new file mode 100644
index 0000000..985c23e
--- /dev/null
+++ b/website/src/app/llms.txt/route.ts
@@ -0,0 +1,34 @@
+import { NextResponse } from 'next/server';
+import { getDocTree, flattenTree } from '@/lib/docs';
+import { appName, appTagline, siteUrl, githubUrl, pypiUrl } from '@/lib/shared';
+
+export const dynamic = 'force-static';
+
+export function GET() {
+ const pages = flattenTree(getDocTree());
+
+ const docLines = pages
+ .map((p) => `- [${p.title}](${new URL(p.url, siteUrl).toString()})`)
+ .join('\n');
+
+ const body = `# ${appName}
+
+> ${appTagline}
+
+${appName} is an open-source CLI that captures browser traffic with Playwright (or Chrome DevTools MCP) and uses your configured AI SDK to generate a typed Python, JavaScript, or TypeScript API client from the recorded HAR.
+
+## Documentation
+
+${docLines}
+
+## Project
+
+- Repository: ${githubUrl}
+- PyPI: ${pypiUrl}
+- License: MIT
+`;
+
+ return new NextResponse(body, {
+ headers: { 'Content-Type': 'text/plain; charset=utf-8' },
+ });
+}
diff --git a/website/src/app/robots.ts b/website/src/app/robots.ts
deleted file mode 100644
index 054f657..0000000
--- a/website/src/app/robots.ts
+++ /dev/null
@@ -1,14 +0,0 @@
-import type { MetadataRoute } from 'next';
-import { siteUrl } from '@/lib/shared';
-
-export const dynamic = 'force-static';
-
-export default function robots(): MetadataRoute.Robots {
- return {
- rules: {
- userAgent: '*',
- allow: '/',
- },
- sitemap: new URL('/sitemap.xml', siteUrl).toString(),
- };
-}
diff --git a/website/src/app/robots.txt/route.ts b/website/src/app/robots.txt/route.ts
new file mode 100644
index 0000000..033ec21
--- /dev/null
+++ b/website/src/app/robots.txt/route.ts
@@ -0,0 +1,22 @@
+import { NextResponse } from 'next/server';
+import { siteUrl } from '@/lib/shared';
+
+export const dynamic = 'force-static';
+
+export function GET() {
+ const sitemap = new URL('/sitemap.xml', siteUrl).toString();
+
+ // Content-Signal: AI-preferences draft (contentsignals.org).
+ // rae is MIT-licensed open source — we want to be trained on, indexed,
+ // and cited by AI tools.
+ const body = `User-agent: *
+Allow: /
+Content-Signal: ai-train=yes, search=yes, ai-input=yes
+
+Sitemap: ${sitemap}
+`;
+
+ return new NextResponse(body, {
+ headers: { 'Content-Type': 'text/plain; charset=utf-8' },
+ });
+}
diff --git a/website/src/components/docs/mdx/tabs.tsx b/website/src/components/docs/mdx/tabs.tsx
index 158113b..9d84417 100644
--- a/website/src/components/docs/mdx/tabs.tsx
+++ b/website/src/components/docs/mdx/tabs.tsx
@@ -39,7 +39,7 @@ export function Tabs({ items, children }: TabsProps) {
<div className="mt-5">
<div
role="tablist"
- className="flex items-center gap-5 border-b border-ink/10 overflow-x-auto"
+ className="flex items-center gap-5 border-b border-ink/10 overflow-x-auto overflow-y-hidden"
>
{tabs.map((t) => {
const isActive = t.value === active;
diff --git a/website/src/components/json-ld.tsx b/website/src/components/json-ld.tsx
new file mode 100644
index 0000000..266c131
--- /dev/null
+++ b/website/src/components/json-ld.tsx
@@ -0,0 +1,16 @@
+/* Schema.org JSON-LD block for SEO + LLM crawler signals.
+ Must render as a plain <script> tag in the static HTML (not via
+ next/script, which only injects through React hydration and so is
+ invisible to non-JS crawlers). The `<` escape prevents a `</script>`
+ sequence from inside a string value breaking out of the tag. */
+
+export function JsonLd({ data }: { data: unknown }) {
+ const json = JSON.stringify(data).replace(/</g, '\\u003c');
+ return (
+ <script
+ type="application/ld+json"
+ // eslint-disable-next-line react/no-danger
+ dangerouslySetInnerHTML={{ __html: json }}
+ />
+ );
+}
diff --git a/website/src/lib/shared.ts b/website/src/lib/shared.ts
index e25bc51..0ac1edb 100644
--- a/website/src/lib/shared.ts
+++ b/website/src/lib/shared.ts
@@ -1,5 +1,5 @@
export const appName = 'Reverse API Engineer';
-export const appTagline = 'Generate typed API clients from browser traffic.';
+export const appTagline = 'Turn websites into APIs.';
export const siteUrl = process.env.NEXT_PUBLIC_SITE_URL ?? 'https://reverseapi.dev';
export const docsRoute = '/docs';
export const docsImageRoute = '/og/docs';