Skip to content

The agent that turns websites into APIs!

feat(website): SEO metadata + sitemap/robots + recover lib/ files

kalil0321c1a675b2bf24
diff --git a/.gitignore b/.gitignore
index 01f55d2..de2c13f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,8 @@ eggs/
.eggs/
lib/
lib64/
+!website/src/lib/
+!website/src/lib/**
parts/
sdist/
var/
@@ -102,4 +104,4 @@ debug/
*.zip
store-assets/
-.playwright-mcp/
\ No newline at end of file
+.playwright-mcp/
diff --git a/website/content/docs/index.mdx b/website/content/docs/index.mdx
index ca0f23c..a9315a0 100644
--- a/website/content/docs/index.mdx
+++ b/website/content/docs/index.mdx
@@ -31,6 +31,21 @@ can map that backend for you. The generated client wraps the same endpoints
your browser called, with auth, pagination, and error handling already wired
up.
+### Where it fits
+
+Use it when the browser already knows the workflow, but the API is undocumented
+or too time-consuming to map by hand:
+
+- **Job boards**: capture search, filters, pagination, and detail endpoints.
+- **E-commerce sites**: turn listing and product requests into reusable client
+ methods.
+- **Authenticated dashboards**: use manual capture or Chrome DevTools MCP when
+ cookies and existing sessions matter.
+- **Internal tools**: document backend calls before replacing brittle UI
+ automation.
+- **Agent workflows**: run captures from another agent and consume the JSON
+ result programmatically.
+
### Drive captures from another agent
Scripted commands are available for agent workflows. `agent` and `engineer`
diff --git a/website/src/app/(home)/page.tsx b/website/src/app/(home)/page.tsx
index f59a229..ae58b75 100644
--- a/website/src/app/(home)/page.tsx
+++ b/website/src/app/(home)/page.tsx
@@ -1,10 +1,26 @@
import Link from 'next/link';
+import type { Metadata } from 'next';
import { ArrowRightIcon } from 'lucide-react';
import { githubUrl } from '@/lib/shared';
import { HeroCyclingPhrase } from '@/components/hero-cycling-phrase';
import { InstallCommand } from '@/components/install-command';
import { BuiltInTheOpen } from '@/components/built-in-the-open';
+export const metadata: Metadata = {
+ title: 'Generate API Clients from Browser Traffic',
+ description:
+ 'Capture browser traffic with Playwright or Chrome DevTools MCP and generate typed Python, JavaScript, or TypeScript API clients.',
+ alternates: {
+ canonical: '/',
+ },
+ openGraph: {
+ title: 'Generate API Clients from Browser Traffic',
+ description:
+ 'Capture browser traffic and turn HAR recordings into typed API clients.',
+ url: '/',
+ },
+};
+
function GithubIcon({ className }: { className?: string }) {
return (
<svg viewBox="0 0 24 24" fill="currentColor" className={className} aria-hidden="true">
@@ -95,7 +111,7 @@ function HowItWorks() {
const steps = [
{ num: '01', title: 'Browse', body: 'Open the CLI. Drive the browser, or let the agent.' },
{ num: '02', title: 'Capture', body: 'HAR records every request, header, and body.' },
- { num: '03', title: 'Generate', body: 'Claude reads the HAR. Writes a typed client.' },
+ { num: '03', title: 'Generate', body: 'Your model reads the HAR. Writes a typed client.' },
{ num: '04', title: 'Review', body: 'Audit the output, then commit it like any code.' },
];
return (
diff --git a/website/src/app/docs/[[...slug]]/page.tsx b/website/src/app/docs/[[...slug]]/page.tsx
index 99a88a6..0c16575 100644
--- a/website/src/app/docs/[[...slug]]/page.tsx
+++ b/website/src/app/docs/[[...slug]]/page.tsx
@@ -13,6 +13,7 @@ import {
flattenTree,
} from '@/lib/docs';
import { getDocsMdxComponents } from '@/components/docs/mdx-components';
+import { appName } from '@/lib/shared';
interface PageProps {
params: Promise<{ slug?: string[] }>;
@@ -125,7 +126,22 @@ export async function generateMetadata({ params }: PageProps): Promise<Metadata>
const page = getDocPage(slug);
if (!page) return {};
return {
- title: `${page.frontmatter.title} · reverse-api-engineer`,
+ title: page.frontmatter.title,
description: page.frontmatter.description,
+ alternates: {
+ canonical: page.url,
+ },
+ openGraph: {
+ type: 'article',
+ siteName: appName,
+ title: `${page.frontmatter.title} · ${appName}`,
+ description: page.frontmatter.description,
+ url: page.url,
+ },
+ twitter: {
+ card: 'summary',
+ title: `${page.frontmatter.title} · ${appName}`,
+ description: page.frontmatter.description,
+ },
};
}
diff --git a/website/src/app/layout.tsx b/website/src/app/layout.tsx
index 303eb7e..8ed1c5d 100644
--- a/website/src/app/layout.tsx
+++ b/website/src/app/layout.tsx
@@ -1,14 +1,52 @@
import './global.css';
import { Inter, JetBrains_Mono, Fraunces } from 'next/font/google';
import type { Metadata } from 'next';
-import { appName, appTagline } from '@/lib/shared';
+import { appName, appTagline, githubUrl, siteUrl } from '@/lib/shared';
import { Providers } from '@/components/providers';
export const metadata: Metadata = {
+ metadataBase: new URL(siteUrl),
applicationName: appName,
- title: appName,
+ title: {
+ default: `${appName} · Generate API Clients from Browser Traffic`,
+ template: `%s · ${appName}`,
+ },
description: appTagline,
- keywords: ['reverse API', 'API client'],
+ keywords: [
+ 'API reverse engineering',
+ 'HAR capture',
+ 'API client generator',
+ 'browser traffic API',
+ 'typed API client',
+ ],
+ alternates: {
+ canonical: '/',
+ },
+ openGraph: {
+ type: 'website',
+ siteName: appName,
+ title: `${appName} · Generate API Clients from Browser Traffic`,
+ description: appTagline,
+ url: '/',
+ images: [
+ {
+ url: `${githubUrl}/raw/main/assets/reverse-api-banner.jpg`,
+ width: 1200,
+ height: 630,
+ alt: `${appName} banner`,
+ },
+ ],
+ },
+ twitter: {
+ card: 'summary_large_image',
+ title: `${appName} · Generate API Clients from Browser Traffic`,
+ description: appTagline,
+ images: [`${githubUrl}/raw/main/assets/reverse-api-banner.jpg`],
+ },
+ robots: {
+ index: true,
+ follow: true,
+ },
};
const inter = Inter({
diff --git a/website/src/app/robots.ts b/website/src/app/robots.ts
new file mode 100644
index 0000000..054f657
--- /dev/null
+++ b/website/src/app/robots.ts
@@ -0,0 +1,14 @@
+import type { MetadataRoute } from 'next';
+import { siteUrl } from '@/lib/shared';
+
+export const dynamic = 'force-static';
+
+export default function robots(): MetadataRoute.Robots {
+ return {
+ rules: {
+ userAgent: '*',
+ allow: '/',
+ },
+ sitemap: new URL('/sitemap.xml', siteUrl).toString(),
+ };
+}
diff --git a/website/src/app/sitemap.ts b/website/src/app/sitemap.ts
new file mode 100644
index 0000000..d04205b
--- /dev/null
+++ b/website/src/app/sitemap.ts
@@ -0,0 +1,38 @@
+import fs from 'node:fs';
+import type { MetadataRoute } from 'next';
+import { getAllDocPages } from '@/lib/docs';
+import { siteUrl } from '@/lib/shared';
+
+export const dynamic = 'force-static';
+
+function absoluteUrl(path: string) {
+ return new URL(path, siteUrl).toString();
+}
+
+function lastModified(filePath?: string) {
+ if (!filePath) return new Date();
+ try {
+ return fs.statSync(filePath).mtime;
+ } catch {
+ return new Date();
+ }
+}
+
+export default function sitemap(): MetadataRoute.Sitemap {
+ const docs = getAllDocPages().map((page) => ({
+ url: absoluteUrl(page.url),
+ lastModified: lastModified(page.filePath),
+ changeFrequency: 'weekly' as const,
+ priority: page.url === '/docs' ? 0.8 : 0.65,
+ }));
+
+ return [
+ {
+ url: absoluteUrl('/'),
+ lastModified: new Date(),
+ changeFrequency: 'weekly',
+ priority: 1,
+ },
+ ...docs,
+ ];
+}
diff --git a/website/src/lib/cn.ts b/website/src/lib/cn.ts
new file mode 100644
index 0000000..ba66fd2
--- /dev/null
+++ b/website/src/lib/cn.ts
@@ -0,0 +1 @@
+export { twMerge as cn } from 'tailwind-merge';
diff --git a/website/src/lib/docs.ts b/website/src/lib/docs.ts
new file mode 100644
index 0000000..7cff607
--- /dev/null
+++ b/website/src/lib/docs.ts
@@ -0,0 +1,215 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import matter from 'gray-matter';
+
+const DOCS_DIR = path.join(process.cwd(), 'content', 'docs');
+const BASE_URL = '/docs';
+
+// ─── Types ────────────────────────────────────────────────────────────────
+
+export interface DocFrontmatter {
+ title: string;
+ description?: string;
+}
+
+export interface DocPage {
+ slug: string[]; // [], ['installation'], ['modes', 'agent']
+ url: string; // /docs, /docs/installation, /docs/modes/agent
+ filePath: string; // absolute fs path
+ frontmatter: DocFrontmatter;
+ content: string; // raw MDX body (no frontmatter)
+}
+
+export interface TocItem {
+ title: string;
+ url: string; // "#some-slug"
+ depth: number; // 2..4
+}
+
+export type TreeNode =
+ | { type: 'page'; url: string; title: string }
+ | { type: 'folder'; title: string; children: TreeNode[] }
+ | { type: 'separator'; title: string };
+
+// ─── Helpers ──────────────────────────────────────────────────────────────
+
+function readMeta(dir: string): { pages?: string[]; title?: string } | null {
+ const p = path.join(dir, 'meta.json');
+ if (!fs.existsSync(p)) return null;
+ return JSON.parse(fs.readFileSync(p, 'utf8'));
+}
+
+function titleCase(s: string): string {
+ return s.replace(/[-_]/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase());
+}
+
+function isSeparator(name: string): string | null {
+ const m = name.match(/^-{3}(.+?)-{3}$/);
+ return m ? m[1] : null;
+}
+
+// ─── Page loader ──────────────────────────────────────────────────────────
+
+export function getDocPage(slug: string[] | undefined): DocPage | null {
+ const segments = slug ?? [];
+ const candidates: string[] = [];
+
+ if (segments.length === 0) {
+ candidates.push(path.join(DOCS_DIR, 'index.mdx'));
+ } else {
+ candidates.push(path.join(DOCS_DIR, ...segments) + '.mdx');
+ candidates.push(path.join(DOCS_DIR, ...segments, 'index.mdx'));
+ }
+
+ for (const filePath of candidates) {
+ if (fs.existsSync(filePath)) {
+ const raw = fs.readFileSync(filePath, 'utf8');
+ const parsed = matter(raw);
+ return {
+ slug: segments,
+ url: segments.length === 0 ? BASE_URL : `${BASE_URL}/${segments.join('/')}`,
+ filePath,
+ frontmatter: parsed.data as DocFrontmatter,
+ content: parsed.content,
+ };
+ }
+ }
+ return null;
+}
+
+// ─── All pages (for static params) ────────────────────────────────────────
+
+export function getAllDocPages(): DocPage[] {
+ const out: DocPage[] = [];
+ walk(DOCS_DIR, []);
+ return out;
+
+ function walk(dir: string, segments: string[]) {
+ const entries = fs.readdirSync(dir, { withFileTypes: true });
+ for (const entry of entries) {
+ if (entry.name === 'meta.json') continue;
+ const full = path.join(dir, entry.name);
+ if (entry.isDirectory()) {
+ walk(full, [...segments, entry.name]);
+ } else if (entry.name.endsWith('.mdx')) {
+ const base = entry.name.replace(/\.mdx$/, '');
+ const slug = base === 'index' ? segments : [...segments, base];
+ const page = getDocPage(slug);
+ if (page) out.push(page);
+ }
+ }
+ }
+}
+
+// ─── Sidebar tree ─────────────────────────────────────────────────────────
+
+export function getDocTree(): TreeNode[] {
+ return buildTree(DOCS_DIR, []);
+}
+
+function buildTree(dir: string, segments: string[]): TreeNode[] {
+ const meta = readMeta(dir);
+ const entries = fs.readdirSync(dir, { withFileTypes: true });
+
+ const childEntries = entries
+ .filter((e) => e.name !== 'meta.json' && (e.isDirectory() || e.name.endsWith('.mdx')))
+ .map((e) => (e.isDirectory() ? e.name : e.name.replace(/\.mdx$/, '')));
+
+ const order: string[] = meta?.pages
+ ? meta.pages
+ : childEntries.filter((n) => n !== 'index');
+
+ const tree: TreeNode[] = [];
+
+ for (const item of order) {
+ const sep = isSeparator(item);
+ if (sep) {
+ tree.push({ type: 'separator', title: sep.trim() });
+ continue;
+ }
+
+ if (item === 'index') {
+ // Render the folder's own index.mdx as a top-level link
+ const indexPath = path.join(dir, 'index.mdx');
+ if (fs.existsSync(indexPath)) {
+ const fm = matter(fs.readFileSync(indexPath, 'utf8')).data as DocFrontmatter;
+ const url = segments.length === 0 ? BASE_URL : `${BASE_URL}/${segments.join('/')}`;
+ tree.push({
+ type: 'page',
+ url,
+ title: fm.title ?? 'Introduction',
+ });
+ }
+ continue;
+ }
+
+ const mdxPath = path.join(dir, `${item}.mdx`);
+ const folderPath = path.join(dir, item);
+
+ if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {
+ const childTree = buildTree(folderPath, [...segments, item]);
+ const folderMeta = readMeta(folderPath);
+ tree.push({
+ type: 'folder',
+ title: folderMeta?.title ?? titleCase(item),
+ children: childTree,
+ });
+ } else if (fs.existsSync(mdxPath)) {
+ const raw = fs.readFileSync(mdxPath, 'utf8');
+ const fm = matter(raw).data as DocFrontmatter;
+ const slug = [...segments, item];
+ tree.push({
+ type: 'page',
+ url: `${BASE_URL}/${slug.join('/')}`,
+ title: fm.title ?? titleCase(item),
+ });
+ }
+ }
+
+ return tree;
+}
+
+// ─── Flatten tree for prev/next nav ───────────────────────────────────────
+
+export function flattenTree(nodes: TreeNode[]): Array<{ url: string; title: string }> {
+ const out: Array<{ url: string; title: string }> = [];
+ for (const n of nodes) {
+ if (n.type === 'page') out.push({ url: n.url, title: n.title });
+ if (n.type === 'folder') out.push(...flattenTree(n.children));
+ }
+ return out;
+}
+
+// ─── TOC extraction (regex-based, matches rehype-slug) ────────────────────
+
+export function extractToc(mdx: string): TocItem[] {
+ const out: TocItem[] = [];
+ let inFence = false;
+ for (const line of mdx.split('\n')) {
+ if (line.startsWith('```')) {
+ inFence = !inFence;
+ continue;
+ }
+ if (inFence) continue;
+ const m = line.match(/^(#{2,4})\s+(.+?)\s*$/);
+ if (m) {
+ const depth = m[1].length;
+ const title = m[2].replace(/[*_`]/g, '');
+ const slug = slugify(title);
+ out.push({ title, url: `#${slug}`, depth });
+ }
+ }
+ return out;
+}
+
+function slugify(text: string): string {
+ return text
+ .toLowerCase()
+ .normalize('NFKD')
+ .replace(/[̀-ͯ]/g, '')
+ .replace(/&/g, ' and ')
+ .replace(/[^a-z0-9\s-]/g, '')
+ .trim()
+ .replace(/\s+/g, '-')
+ .replace(/-+/g, '-');
+}
diff --git a/website/src/lib/shared.ts b/website/src/lib/shared.ts
new file mode 100644
index 0000000..e25bc51
--- /dev/null
+++ b/website/src/lib/shared.ts
@@ -0,0 +1,15 @@
+export const appName = 'Reverse API Engineer';
+export const appTagline = 'Generate typed API clients from browser traffic.';
+export const siteUrl = process.env.NEXT_PUBLIC_SITE_URL ?? 'https://reverseapi.dev';
+export const docsRoute = '/docs';
+export const docsImageRoute = '/og/docs';
+export const docsContentRoute = '/llms.mdx/docs';
+
+export const gitConfig = {
+ user: 'kalil0321',
+ repo: 'reverse-api-engineer',
+ branch: 'main',
+};
+
+export const githubUrl = `https://github.com/${gitConfig.user}/${gitConfig.repo}`;
+export const pypiUrl = 'https://pypi.org/project/reverse-api-engineer/';