The agent that turns websites into APIs!
feat(website): SEO metadata + sitemap/robots + recover lib/ files
kalil0321c1a675b2bf24
diff --git a/.gitignore b/.gitignoreindex 01f55d2..de2c13f 100644--- a/.gitignore+++ b/.gitignore@@ -12,6 +12,8 @@ eggs/.eggs/lib/lib64/+!website/src/lib/+!website/src/lib/**parts/sdist/var/@@ -102,4 +104,4 @@ debug/*.zipstore-assets/-.playwright-mcp/\ No newline at end of file+.playwright-mcp/diff --git a/website/content/docs/index.mdx b/website/content/docs/index.mdxindex ca0f23c..a9315a0 100644--- a/website/content/docs/index.mdx+++ b/website/content/docs/index.mdx@@ -31,6 +31,21 @@ can map that backend for you. The generated client wraps the same endpointsyour browser called, with auth, pagination, and error handling already wiredup.+### Where it fits++Use it when the browser already knows the workflow, but the API is undocumented+or too time-consuming to map by hand:++- **Job boards**: capture search, filters, pagination, and detail endpoints.+- **E-commerce sites**: turn listing and product requests into reusable client+ methods.+- **Authenticated dashboards**: use manual capture or Chrome DevTools MCP when+ cookies and existing sessions matter.+- **Internal tools**: document backend calls before replacing brittle UI+ automation.+- **Agent workflows**: run captures from another agent and consume the JSON+ result programmatically.+### Drive captures from another agentScripted commands are available for agent workflows. `agent` and `engineer`diff --git a/website/src/app/(home)/page.tsx b/website/src/app/(home)/page.tsxindex f59a229..ae58b75 100644--- a/website/src/app/(home)/page.tsx+++ b/website/src/app/(home)/page.tsx@@ -1,10 +1,26 @@import Link from 'next/link';+import type { Metadata } from 'next';import { ArrowRightIcon } from 'lucide-react';import { githubUrl } from '@/lib/shared';import { HeroCyclingPhrase } from '@/components/hero-cycling-phrase';import { InstallCommand } from '@/components/install-command';import { BuiltInTheOpen } from '@/components/built-in-the-open';+export const metadata: Metadata = {+ title: 'Generate API Clients from Browser Traffic',+ description:+ 'Capture browser traffic with Playwright or Chrome DevTools MCP and generate typed Python, JavaScript, or TypeScript API clients.',+ alternates: {+ canonical: '/',+ },+ openGraph: {+ title: 'Generate API Clients from Browser Traffic',+ description:+ 'Capture browser traffic and turn HAR recordings into typed API clients.',+ url: '/',+ },+};+function GithubIcon({ className }: { className?: string }) {return (<svg viewBox="0 0 24 24" fill="currentColor" className={className} aria-hidden="true">@@ -95,7 +111,7 @@ function HowItWorks() {const steps = [{ num: '01', title: 'Browse', body: 'Open the CLI. Drive the browser, or let the agent.' },{ num: '02', title: 'Capture', body: 'HAR records every request, header, and body.' },- { num: '03', title: 'Generate', body: 'Claude reads the HAR. Writes a typed client.' },+ { num: '03', title: 'Generate', body: 'Your model reads the HAR. Writes a typed client.' },{ num: '04', title: 'Review', body: 'Audit the output, then commit it like any code.' },];return (diff --git a/website/src/app/docs/[[...slug]]/page.tsx b/website/src/app/docs/[[...slug]]/page.tsxindex 99a88a6..0c16575 100644--- a/website/src/app/docs/[[...slug]]/page.tsx+++ b/website/src/app/docs/[[...slug]]/page.tsx@@ -13,6 +13,7 @@ import {flattenTree,} from '@/lib/docs';import { getDocsMdxComponents } from '@/components/docs/mdx-components';+import { appName } from '@/lib/shared';interface PageProps {params: Promise<{ slug?: string[] }>;@@ -125,7 +126,22 @@ export async function generateMetadata({ params }: PageProps): Promise<Metadata>const page = getDocPage(slug);if (!page) return {};return {- title: `${page.frontmatter.title} · reverse-api-engineer`,+ title: page.frontmatter.title,description: page.frontmatter.description,+ alternates: {+ canonical: page.url,+ },+ openGraph: {+ type: 'article',+ siteName: appName,+ title: `${page.frontmatter.title} · ${appName}`,+ description: page.frontmatter.description,+ url: page.url,+ },+ twitter: {+ card: 'summary',+ title: `${page.frontmatter.title} · ${appName}`,+ description: page.frontmatter.description,+ },};}diff --git a/website/src/app/layout.tsx b/website/src/app/layout.tsxindex 303eb7e..8ed1c5d 100644--- a/website/src/app/layout.tsx+++ b/website/src/app/layout.tsx@@ -1,14 +1,52 @@import './global.css';import { Inter, JetBrains_Mono, Fraunces } from 'next/font/google';import type { Metadata } from 'next';-import { appName, appTagline } from '@/lib/shared';+import { appName, appTagline, githubUrl, siteUrl } from '@/lib/shared';import { Providers } from '@/components/providers';export const metadata: Metadata = {+ metadataBase: new URL(siteUrl),applicationName: appName,- title: appName,+ title: {+ default: `${appName} · Generate API Clients from Browser Traffic`,+ template: `%s · ${appName}`,+ },description: appTagline,- keywords: ['reverse API', 'API client'],+ keywords: [+ 'API reverse engineering',+ 'HAR capture',+ 'API client generator',+ 'browser traffic API',+ 'typed API client',+ ],+ alternates: {+ canonical: '/',+ },+ openGraph: {+ type: 'website',+ siteName: appName,+ title: `${appName} · Generate API Clients from Browser Traffic`,+ description: appTagline,+ url: '/',+ images: [+ {+ url: `${githubUrl}/raw/main/assets/reverse-api-banner.jpg`,+ width: 1200,+ height: 630,+ alt: `${appName} banner`,+ },+ ],+ },+ twitter: {+ card: 'summary_large_image',+ title: `${appName} · Generate API Clients from Browser Traffic`,+ description: appTagline,+ images: [`${githubUrl}/raw/main/assets/reverse-api-banner.jpg`],+ },+ robots: {+ index: true,+ follow: true,+ },};const inter = Inter({diff --git a/website/src/app/robots.ts b/website/src/app/robots.tsnew file mode 100644index 0000000..054f657--- /dev/null+++ b/website/src/app/robots.ts@@ -0,0 +1,14 @@+import type { MetadataRoute } from 'next';+import { siteUrl } from '@/lib/shared';++export const dynamic = 'force-static';++export default function robots(): MetadataRoute.Robots {+ return {+ rules: {+ userAgent: '*',+ allow: '/',+ },+ sitemap: new URL('/sitemap.xml', siteUrl).toString(),+ };+}diff --git a/website/src/app/sitemap.ts b/website/src/app/sitemap.tsnew file mode 100644index 0000000..d04205b--- /dev/null+++ b/website/src/app/sitemap.ts@@ -0,0 +1,38 @@+import fs from 'node:fs';+import type { MetadataRoute } from 'next';+import { getAllDocPages } from '@/lib/docs';+import { siteUrl } from '@/lib/shared';++export const dynamic = 'force-static';++function absoluteUrl(path: string) {+ return new URL(path, siteUrl).toString();+}++function lastModified(filePath?: string) {+ if (!filePath) return new Date();+ try {+ return fs.statSync(filePath).mtime;+ } catch {+ return new Date();+ }+}++export default function sitemap(): MetadataRoute.Sitemap {+ const docs = getAllDocPages().map((page) => ({+ url: absoluteUrl(page.url),+ lastModified: lastModified(page.filePath),+ changeFrequency: 'weekly' as const,+ priority: page.url === '/docs' ? 0.8 : 0.65,+ }));++ return [+ {+ url: absoluteUrl('/'),+ lastModified: new Date(),+ changeFrequency: 'weekly',+ priority: 1,+ },+ ...docs,+ ];+}diff --git a/website/src/lib/cn.ts b/website/src/lib/cn.tsnew file mode 100644index 0000000..ba66fd2--- /dev/null+++ b/website/src/lib/cn.ts@@ -0,0 +1 @@+export { twMerge as cn } from 'tailwind-merge';diff --git a/website/src/lib/docs.ts b/website/src/lib/docs.tsnew file mode 100644index 0000000..7cff607--- /dev/null+++ b/website/src/lib/docs.ts@@ -0,0 +1,215 @@+import fs from 'node:fs';+import path from 'node:path';+import matter from 'gray-matter';++const DOCS_DIR = path.join(process.cwd(), 'content', 'docs');+const BASE_URL = '/docs';++// ─── Types ────────────────────────────────────────────────────────────────++export interface DocFrontmatter {+ title: string;+ description?: string;+}++export interface DocPage {+ slug: string[]; // [], ['installation'], ['modes', 'agent']+ url: string; // /docs, /docs/installation, /docs/modes/agent+ filePath: string; // absolute fs path+ frontmatter: DocFrontmatter;+ content: string; // raw MDX body (no frontmatter)+}++export interface TocItem {+ title: string;+ url: string; // "#some-slug"+ depth: number; // 2..4+}++export type TreeNode =+ | { type: 'page'; url: string; title: string }+ | { type: 'folder'; title: string; children: TreeNode[] }+ | { type: 'separator'; title: string };++// ─── Helpers ──────────────────────────────────────────────────────────────++function readMeta(dir: string): { pages?: string[]; title?: string } | null {+ const p = path.join(dir, 'meta.json');+ if (!fs.existsSync(p)) return null;+ return JSON.parse(fs.readFileSync(p, 'utf8'));+}++function titleCase(s: string): string {+ return s.replace(/[-_]/g, ' ').replace(/\b\w/g, (c) => c.toUpperCase());+}++function isSeparator(name: string): string | null {+ const m = name.match(/^-{3}(.+?)-{3}$/);+ return m ? m[1] : null;+}++// ─── Page loader ──────────────────────────────────────────────────────────++export function getDocPage(slug: string[] | undefined): DocPage | null {+ const segments = slug ?? [];+ const candidates: string[] = [];++ if (segments.length === 0) {+ candidates.push(path.join(DOCS_DIR, 'index.mdx'));+ } else {+ candidates.push(path.join(DOCS_DIR, ...segments) + '.mdx');+ candidates.push(path.join(DOCS_DIR, ...segments, 'index.mdx'));+ }++ for (const filePath of candidates) {+ if (fs.existsSync(filePath)) {+ const raw = fs.readFileSync(filePath, 'utf8');+ const parsed = matter(raw);+ return {+ slug: segments,+ url: segments.length === 0 ? BASE_URL : `${BASE_URL}/${segments.join('/')}`,+ filePath,+ frontmatter: parsed.data as DocFrontmatter,+ content: parsed.content,+ };+ }+ }+ return null;+}++// ─── All pages (for static params) ────────────────────────────────────────++export function getAllDocPages(): DocPage[] {+ const out: DocPage[] = [];+ walk(DOCS_DIR, []);+ return out;++ function walk(dir: string, segments: string[]) {+ const entries = fs.readdirSync(dir, { withFileTypes: true });+ for (const entry of entries) {+ if (entry.name === 'meta.json') continue;+ const full = path.join(dir, entry.name);+ if (entry.isDirectory()) {+ walk(full, [...segments, entry.name]);+ } else if (entry.name.endsWith('.mdx')) {+ const base = entry.name.replace(/\.mdx$/, '');+ const slug = base === 'index' ? segments : [...segments, base];+ const page = getDocPage(slug);+ if (page) out.push(page);+ }+ }+ }+}++// ─── Sidebar tree ─────────────────────────────────────────────────────────++export function getDocTree(): TreeNode[] {+ return buildTree(DOCS_DIR, []);+}++function buildTree(dir: string, segments: string[]): TreeNode[] {+ const meta = readMeta(dir);+ const entries = fs.readdirSync(dir, { withFileTypes: true });++ const childEntries = entries+ .filter((e) => e.name !== 'meta.json' && (e.isDirectory() || e.name.endsWith('.mdx')))+ .map((e) => (e.isDirectory() ? e.name : e.name.replace(/\.mdx$/, '')));++ const order: string[] = meta?.pages+ ? meta.pages+ : childEntries.filter((n) => n !== 'index');++ const tree: TreeNode[] = [];++ for (const item of order) {+ const sep = isSeparator(item);+ if (sep) {+ tree.push({ type: 'separator', title: sep.trim() });+ continue;+ }++ if (item === 'index') {+ // Render the folder's own index.mdx as a top-level link+ const indexPath = path.join(dir, 'index.mdx');+ if (fs.existsSync(indexPath)) {+ const fm = matter(fs.readFileSync(indexPath, 'utf8')).data as DocFrontmatter;+ const url = segments.length === 0 ? BASE_URL : `${BASE_URL}/${segments.join('/')}`;+ tree.push({+ type: 'page',+ url,+ title: fm.title ?? 'Introduction',+ });+ }+ continue;+ }++ const mdxPath = path.join(dir, `${item}.mdx`);+ const folderPath = path.join(dir, item);++ if (fs.existsSync(folderPath) && fs.statSync(folderPath).isDirectory()) {+ const childTree = buildTree(folderPath, [...segments, item]);+ const folderMeta = readMeta(folderPath);+ tree.push({+ type: 'folder',+ title: folderMeta?.title ?? titleCase(item),+ children: childTree,+ });+ } else if (fs.existsSync(mdxPath)) {+ const raw = fs.readFileSync(mdxPath, 'utf8');+ const fm = matter(raw).data as DocFrontmatter;+ const slug = [...segments, item];+ tree.push({+ type: 'page',+ url: `${BASE_URL}/${slug.join('/')}`,+ title: fm.title ?? titleCase(item),+ });+ }+ }++ return tree;+}++// ─── Flatten tree for prev/next nav ───────────────────────────────────────++export function flattenTree(nodes: TreeNode[]): Array<{ url: string; title: string }> {+ const out: Array<{ url: string; title: string }> = [];+ for (const n of nodes) {+ if (n.type === 'page') out.push({ url: n.url, title: n.title });+ if (n.type === 'folder') out.push(...flattenTree(n.children));+ }+ return out;+}++// ─── TOC extraction (regex-based, matches rehype-slug) ────────────────────++export function extractToc(mdx: string): TocItem[] {+ const out: TocItem[] = [];+ let inFence = false;+ for (const line of mdx.split('\n')) {+ if (line.startsWith('```')) {+ inFence = !inFence;+ continue;+ }+ if (inFence) continue;+ const m = line.match(/^(#{2,4})\s+(.+?)\s*$/);+ if (m) {+ const depth = m[1].length;+ const title = m[2].replace(/[*_`]/g, '');+ const slug = slugify(title);+ out.push({ title, url: `#${slug}`, depth });+ }+ }+ return out;+}++function slugify(text: string): string {+ return text+ .toLowerCase()+ .normalize('NFKD')+ .replace(/[̀-ͯ]/g, '')+ .replace(/&/g, ' and ')+ .replace(/[^a-z0-9\s-]/g, '')+ .trim()+ .replace(/\s+/g, '-')+ .replace(/-+/g, '-');+}diff --git a/website/src/lib/shared.ts b/website/src/lib/shared.tsnew file mode 100644index 0000000..e25bc51--- /dev/null+++ b/website/src/lib/shared.ts@@ -0,0 +1,15 @@+export const appName = 'Reverse API Engineer';+export const appTagline = 'Generate typed API clients from browser traffic.';+export const siteUrl = process.env.NEXT_PUBLIC_SITE_URL ?? 'https://reverseapi.dev';+export const docsRoute = '/docs';+export const docsImageRoute = '/og/docs';+export const docsContentRoute = '/llms.mdx/docs';++export const gitConfig = {+ user: 'kalil0321',+ repo: 'reverse-api-engineer',+ branch: 'main',+};++export const githubUrl = `https://github.com/${gitConfig.user}/${gitConfig.repo}`;+export const pypiUrl = 'https://pypi.org/project/reverse-api-engineer/';