Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(seo): make entity pages readable to a crawler and stop submitting the untouched ones
nirholas37b189ffef20
diff --git a/api/_lib/crawler-page.js b/api/_lib/crawler-page.jsnew file mode 100644index 000000000..050db9da3--- /dev/null+++ b/api/_lib/crawler-page.js@@ -0,0 +1,274 @@+/**+ * Server-rendered entity pages for crawlers.+ * -----------------------------------------+ * /avatars/:id and /agents/:id are single-page-app routes: the HTML a browser+ * receives is an empty shell and the entity is fetched client-side. vercel.json+ * routes known bot User-Agents to an SSR handler instead so a shared link+ * unfurls, and so a search engine has something to index.+ *+ * The first version of that SSR page carried only <meta> tags: the visible body+ * was a spinner, the words lived inside <noscript>, and the last line was+ * `location.replace(<this same URL>)`. That combination is invisible to a+ * search engine in the worst possible way:+ *+ * - A rendering crawler (Googlebot, bingbot, Applebot) runs the script. The+ * replace() targets the URL it is already on and the UA still matches the+ * bot branch, so the response is byte-identical and the navigation repeats:+ * a self-reload loop, and a URL the crawler may bucket as a redirect.+ * - Rendering also drops <noscript>, so the only real text on the page is+ * removed before indexing. Every entity URL then renders as the same+ * spinner over the same "Loading…" line, i.e. one duplicate cluster tens of+ * thousands of pages wide with no unique content to tell them apart.+ *+ * So the renderers here put the entity's real content in the body, and the+ * self-redirect is emitted only for crawlers that never index (link unfurlers+ * like Embedly or Iframely, where bouncing a human who arrives with a scraper+ * UA is still the friendly outcome). Search engines get content and no script.+ *+ * Content parity is what keeps this out of cloaking territory: a crawler sees+ * the same name, description, tags, owner, and imagery the interactive page+ * shows, minus the WebGL viewport it cannot run.+ */++// Crawlers that index (and, for the big three, render JS before indexing).+// These must never be handed a self-redirect or a content-free body.+const SEARCH_CRAWLERS =+ /(Googlebot|Google-InspectionTool|GoogleOther|Storebot-Google|Google-Extended|bingbot|BingPreview|Applebot|DuckDuckBot|DuckAssistBot|YandexBot|Baiduspider|PetalBot|SeznamBot|Amazonbot|Bytespider|CCBot|GPTBot|OAI-SearchBot|ChatGPT-User|ClaudeBot|Claude-SearchBot|Claude-User|PerplexityBot|Perplexity-User|MistralAI-User|cohere-ai|YouBot)/i;++/**+ * True when this User-Agent belongs to a crawler that builds a search or+ * answer index, as opposed to a scraper that only wants unfurl metadata.+ *+ * @param {string|undefined|null} ua+ * @returns {boolean}+ */+export function isSearchCrawler(ua) {+ return SEARCH_CRAWLERS.test(String(ua || ''));+}++/** HTML-escape a value for use in text or a double-quoted attribute. */+export function esc(s) {+ return String(s ?? '').replace(+ /[&<>"']/g,+ (c) => ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[c],+ );+}++// One stylesheet for every crawler page. Inline because the page must be+// self-contained: it is served from an /api route and a crawler that fetches+// no subresources should still get a page that reads correctly.+const STYLES = `+ :root{color-scheme:dark}+ *,*::before,*::after{box-sizing:border-box}+ html,body{margin:0;padding:0;background:#06070a;color:#e7e9ee;+ font:16px/1.6 Inter,system-ui,-apple-system,"Segoe UI",sans-serif;+ -webkit-font-smoothing:antialiased}+ a{color:#9fd0ff;text-decoration:none}+ a:hover,a:focus-visible{text-decoration:underline;text-underline-offset:3px}+ a:focus-visible{outline:2px solid #9fd0ff;outline-offset:3px;border-radius:4px}+ main{max-width:56rem;margin:0 auto;padding:3rem 1.25rem 4rem}+ nav[aria-label="Breadcrumb"]{font-size:.8125rem;color:#8b93a7;margin-bottom:1.5rem}+ nav[aria-label="Breadcrumb"] ol{list-style:none;display:flex;flex-wrap:wrap;gap:.5rem;margin:0;padding:0}+ nav[aria-label="Breadcrumb"] li+li::before{content:"/";margin-right:.5rem;color:#4a5164}+ h1{font-size:clamp(1.75rem,4vw,2.5rem);line-height:1.15;margin:0 0 .75rem;letter-spacing:-.02em}+ .lede{font-size:1.0625rem;color:#b6bccb;margin:0 0 2rem;max-width:44rem}+ .card{display:block;width:100%;max-width:40rem;height:auto;aspect-ratio:1200/630;+ border:1px solid #1d2230;border-radius:14px;background:#0b0d13;margin:0 0 2rem}+ dl{display:grid;grid-template-columns:minmax(6rem,10rem) 1fr;gap:.5rem 1.5rem;margin:0 0 2rem;+ font-size:.9375rem}+ dt{color:#8b93a7}+ dd{margin:0;color:#e7e9ee}+ ul.tags{list-style:none;display:flex;flex-wrap:wrap;gap:.5rem;margin:0;padding:0}+ ul.tags a{display:inline-block;padding:.15rem .6rem;border:1px solid #232838;border-radius:999px;+ font-size:.8125rem;color:#b6bccb}+ .actions{display:flex;flex-wrap:wrap;gap:.75rem;margin:0 0 2.5rem}+ .actions a{display:inline-block;padding:.6rem 1.1rem;border-radius:10px;border:1px solid #232838;+ background:#0e1119;color:#e7e9ee;font-size:.9375rem}+ .actions a.primary{background:#e7e9ee;color:#06070a;border-color:#e7e9ee;font-weight:600}+ footer{border-top:1px solid #171b26;padding-top:1.5rem;font-size:.875rem;color:#8b93a7}+ footer ul{list-style:none;display:flex;flex-wrap:wrap;gap:1rem;margin:.5rem 0 0;padding:0}+ @media(max-width:36rem){dl{grid-template-columns:1fr;gap:.15rem}dt{margin-top:.75rem}}+`;++/** Breadcrumb markup plus the matching schema.org ListItem entries. */+function breadcrumb(trail, origin) {+ const items = trail+ .map((c, i) => {+ const label = esc(c.name);+ const inner = i === trail.length - 1 ? `<span aria-current="page">${label}</span>` : `<a href="${esc(c.path)}">${label}</a>`;+ return `<li>${inner}</li>`;+ })+ .join('');+ const ld = trail.map((c, i) => ({+ '@type': 'ListItem',+ position: i + 1,+ name: c.name,+ item: `${origin}${c.path}`,+ }));+ return { html: `<nav aria-label="Breadcrumb"><ol>${items}</ol></nav>`, ld };+}++/**+ * Render a crawler-facing entity page: real content in the body, self-canonical+ * in the head, structured data, and no self-redirect for an indexing crawler.+ *+ * @param {object} o+ * @param {string} o.title entity name, unescaped+ * @param {string} o.desc one-paragraph summary, unescaped+ * @param {string} o.pageUrl absolute canonical URL+ * @param {string} o.ogImage absolute URL of the 1200x630 card+ * @param {string} o.origin site origin+ * @param {string} o.ogType Open Graph object type+ * @param {string} o.frameButton Farcaster frame button label+ * @param {{name:string,path:string}[]} o.trail breadcrumb, root first+ * @param {{term:string,detail:string}[]} [o.facts] definition rows, unescaped+ * @param {{label:string,href:string}[]} [o.tags] tag chips+ * @param {{label:string,href:string,primary?:boolean}[]} [o.actions] links out+ * @param {{label:string,href:string}[]} [o.related] footer links+ * @param {object} [o.jsonLd] entity node merged into the @graph+ * @param {boolean} [o.redirect] emit the bounce-to-SPA script+ * @param {boolean} [o.noindex] ask crawlers not to index this entity+ * @returns {string} complete HTML document+ */+export function renderCrawlerPage({+ title,+ desc,+ pageUrl,+ ogImage,+ origin,+ ogType = 'website',+ frameButton = 'Open on three.ws',+ trail = [],+ facts = [],+ tags = [],+ actions = [],+ related = [],+ jsonLd = null,+ redirect = false,+ noindex = false,+}) {+ const t = esc(title);+ const d = esc(desc);+ const crumbs = breadcrumb(trail, origin);+ const path = new URL(pageUrl).pathname;++ const graph = [];+ if (jsonLd) graph.push(jsonLd);+ graph.push({ '@type': 'BreadcrumbList', itemListElement: crumbs.ld });++ const factsHtml = facts.length+ ? `<dl>${facts.map((f) => `<dt>${esc(f.term)}</dt><dd>${f.detail}</dd>`).join('')}</dl>`+ : '';+ const tagsHtml = tags.length+ ? `<ul class="tags">${tags.map((g) => `<li><a href="${esc(g.href)}">${esc(g.label)}</a></li>`).join('')}</ul>`+ : '';+ const actionsHtml = actions.length+ ? `<div class="actions">${actions+ .map((a) => `<a href="${esc(a.href)}"${a.primary ? ' class="primary"' : ''}>${esc(a.label)}</a>`)+ .join('')}</div>`+ : '';+ const relatedHtml = related.length+ ? `<ul>${related.map((r) => `<li><a href="${esc(r.href)}">${esc(r.label)}</a></li>`).join('')}</ul>`+ : '';++ return `<!doctype html>+<html lang="en">+<head>+ <meta charset="utf-8">+ <title>${t} · three.ws</title>+ <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover">+ <meta name="description" content="${d}">+ <meta name="theme-color" content="#06070a">+ <meta name="robots" content="${noindex ? 'noindex, follow' : 'index, follow, max-image-preview:large, max-snippet:-1'}">++ <meta property="og:type" content="${esc(ogType)}">+ <meta property="og:site_name" content="three.ws">+ <meta property="og:title" content="${t} · three.ws">+ <meta property="og:description" content="${d}">+ <meta property="og:url" content="${esc(pageUrl)}">+ <meta property="og:image" content="${esc(ogImage)}">+ <meta property="og:image:width" content="1200">+ <meta property="og:image:height" content="630">+ <meta property="og:image:alt" content="${t} on three.ws">++ <meta name="twitter:card" content="summary_large_image">+ <meta name="twitter:title" content="${t} · three.ws">+ <meta name="twitter:description" content="${d}">+ <meta name="twitter:image" content="${esc(ogImage)}">+ <meta name="twitter:creator" content="@trythreews">++ <meta property="fc:frame" content="vNext">+ <meta property="fc:frame:image" content="${esc(ogImage)}">+ <meta property="fc:frame:image:aspect_ratio" content="1.91:1">+ <meta property="fc:frame:button:1" content="${esc(frameButton)}">+ <meta property="fc:frame:button:1:action" content="link">+ <meta property="fc:frame:button:1:target" content="${esc(pageUrl)}">++ ${noindex ? '' : `<link rel="canonical" href="${esc(pageUrl)}">`}+ <link rel="shortcut icon" href="/favicon.ico">+ <style>${STYLES}</style>+ <script type="application/ld+json">${JSON.stringify({ '@context': 'https://schema.org', '@graph': graph }).replace(/</g, '\\u003c')}</script>+</head>+<body>+ <main>+ ${crumbs.html}+ <h1>${t}</h1>+ <p class="lede">${d}</p>+ <img class="card" src="${esc(ogImage)}" width="1200" height="630" alt="${t} on three.ws" loading="eager">+ ${actionsHtml}+ ${factsHtml}+ ${tagsHtml}+ <footer>+ <p>${t} lives on three.ws, the agent layer for the open web: build a 3D avatar, give it an identity, and embed it anywhere.</p>+ ${relatedHtml}+ </footer>+ </main>+${redirect ? ` <script>(function(){window.location.replace(${JSON.stringify(path)});})()</script>\n` : ''}</body>+</html>`;+}++/**+ * Render the page a crawler gets for an id that names nothing public: gone,+ * private, or never existed. Served with a 404 so the URL leaves the index+ * instead of being logged as a redirect to an unrelated listing.+ *+ * @param {object} o+ * @param {string} o.heading unescaped+ * @param {string} o.message unescaped+ * @param {string} o.origin+ * @param {{label:string,href:string}[]} o.actions+ * @param {string} [o.redirect] path to bounce a non-indexing scraper to+ * @returns {string} complete HTML document+ */+export function renderCrawlerNotFound({ heading, message, origin, actions = [], redirect = '' }) {+ const h = esc(heading);+ const m = esc(message);+ return `<!doctype html>+<html lang="en">+<head>+ <meta charset="utf-8">+ <title>${h} · three.ws</title>+ <meta name="viewport" content="width=device-width, initial-scale=1.0">+ <meta name="description" content="${m}">+ <meta name="robots" content="noindex, follow">+ <meta name="theme-color" content="#06070a">+ <meta property="og:type" content="website">+ <meta property="og:site_name" content="three.ws">+ <meta property="og:title" content="${h} · three.ws">+ <meta property="og:description" content="${m}">+ <meta property="og:image" content="${esc(origin)}/og-image.png">+ <meta name="twitter:card" content="summary_large_image">+ <link rel="shortcut icon" href="/favicon.ico">+ <style>${STYLES}</style>+</head>+<body>+ <main>+ <h1>${h}</h1>+ <p class="lede">${m}</p>+ <div class="actions">${actions+ .map((a, i) => `<a href="${esc(a.href)}"${i === 0 ? ' class="primary"' : ''}>${esc(a.label)}</a>`)+ .join('')}</div>+ </main>+${redirect ? ` <script>(function(){window.location.replace(${JSON.stringify(redirect)});})()</script>\n` : ''}</body>+</html>`;+}diff --git a/api/_lib/indexable-entity.js b/api/_lib/indexable-entity.jsnew file mode 100644index 000000000..4fabdec13--- /dev/null+++ b/api/_lib/indexable-entity.js@@ -0,0 +1,90 @@+/**+ * Which avatars and agents are worth asking a search engine to index.+ * ------------------------------------------------------------------+ * Onboarding creates an entity before its owner has said anything about it, and+ * a lot of owners stop there. Measured 2026-09-07: 1,163 of the 3,397 public+ * agents were still named "My First Agent" and 1,158 of those still carried the+ * starter description verbatim. Those pages are not thin, they are *identical*,+ * and no amount of server-side rendering can tell them apart.+ *+ * Submitting them costs twice. They cannot rank, and a duplicate cluster that+ * large drags the pages that could rank down with it (Search Console filed+ * 18,768 URLs under "Duplicate without user-selected canonical" before this).+ *+ * So the untouched ones stay out of the sitemap and their crawler page is served+ * `noindex, follow`. Nothing else changes: the page still renders, still unfurls+ * with the entity's own card, and still links onward. The moment an owner gives+ * it a name or a description it becomes indexable, with no migration and no+ * backfill. That is why the rule reads the row rather than storing a flag.+ *+ * Both the sitemap and the crawler pages import from here so the two can never+ * disagree about what is indexable.+ */++/** Fold case, collapse whitespace, and normalise dash glyphs for comparison. */+function normalize(s) {+ return String(s ?? '')+ .replace(/[\u2010-\u2015]/g, '-')+ .trim()+ .replace(/\s+/g, ' ')+ .toLowerCase();+}++// Names the product hands out when nobody has chosen one. An entity keeping its+// default name is evidence of an untouched row, not of a name someone picked.+const DEFAULT_AGENT_NAMES = new Set([+ 'my first agent',+ 'my agent',+ 'agent',+ 'new agent',+ 'untitled',+ 'untitled agent',+]);++const DEFAULT_AVATAR_NAMES = new Set([+ 'my first agent',+ 'my avatar',+ 'avatar',+ 'new avatar',+ 'untitled',+ 'untitled avatar',+]);++// The description onboarding writes into a brand-new agent. Matched on its+// opening words so a trailing edit, or the dash glyph in the stored copy, does+// not let an untouched row through.+const STARTER_AGENT_DESCRIPTION = 'a friendly starter agent';++/**+ * True when this agent has enough of its own identity to deserve its own entry+ * in a search index.+ *+ * @param {{name?: string|null, description?: string|null}} agent+ * @returns {boolean}+ */+export function isIndexableAgent(agent) {+ const name = normalize(agent?.name);+ if (!name) return false;+ const description = normalize(agent?.description);+ const untouchedName = DEFAULT_AGENT_NAMES.has(name);+ const untouchedDescription = !description || description.startsWith(STARTER_AGENT_DESCRIPTION);+ // Either half being the owner's own work is enough: a custom description+ // under a default name still describes something specific, and vice versa.+ return !(untouchedName && untouchedDescription);+}++/**+ * True when this avatar has enough of its own identity to deserve its own entry+ * in a search index. Avatars carry a generated description and a tag set even+ * when the owner writes nothing, so only a default name with no prose at all+ * counts as untouched.+ *+ * @param {{name?: string|null, description?: string|null, alt_text?: string|null}} avatar+ * @returns {boolean}+ */+export function isIndexableAvatar(avatar) {+ const name = normalize(avatar?.name);+ if (!name) return false;+ if (!DEFAULT_AVATAR_NAMES.has(name)) return true;+ return Boolean(normalize(avatar?.description) || normalize(avatar?.alt_text));+}diff --git a/api/agent-detail-og.js b/api/agent-detail-og.jsindex cad3daa9b..2cbda69fc 100644--- a/api/agent-detail-og.js+++ b/api/agent-detail-og.js@@ -1,148 +1,229 @@/**- * SSR OG page for agent detail (/agents/:id)+ * Crawler page for agent detail (/agents/:id)* -------------------------------------------* GET /api/agent-detail-og?id=<agentId>*- * Wired via vercel.json: when a social crawler hits /agents/<uuid>,- * a User-Agent "has" condition rewrites to this endpoint. Returns a minimal- * HTML page with OG + Twitter Card + Farcaster Frame meta so shared agent- * links unfurl with a real image and description.+ * Wired via vercel.json: when a crawler User-Agent hits /agents/<uuid>, a+ * "has" condition rewrites to this endpoint. Real browsers never reach it, so+ * this is the only version of the page a search engine or a link unfurler ever+ * sees, and it has to carry the agent's real content rather than meta tags over+ * a spinner. See the header of api/_lib/crawler-page.js.*- * Real browsers never reach this route — the rewrite only fires for known- * bot User-Agents. Mirrors the pattern in api/app-og.js.+ * Mirrors api/avatar-detail-og.js.*/import { sql } from './_lib/db.js';import { cors, method, wrap } from './_lib/http.js';import { env } from './_lib/env.js';import { isUuid } from './_lib/validate.js';+import { esc, isSearchCrawler, renderCrawlerPage, renderCrawlerNotFound } from './_lib/crawler-page.js';+import { isIndexableAgent } from './_lib/indexable-entity.js';export default wrap(async (req, res) => {if (cors(req, res, { methods: 'GET,OPTIONS' })) return;- // Crawler-only SSR read. Without this, a POST answered with the 302- // passthrough instead of the 405 the advertised Allow set promises.+ // Crawler-only SSR read. Without this, a POST answered with a page instead+ // of the 405 the advertised Allow set promises.if (!method(req, res, ['GET'])) return;const url = new URL(req.url, 'http://x');const agentId = url.searchParams.get('id');const origin = env.APP_ORIGIN || 'https://three.ws';+ const ua = req.headers?.['user-agent'];+ // Only a non-indexing scraper gets bounced to the app. An indexing crawler+ // runs that script, lands back on this same URL behind the same UA branch,+ // and loops on an identical response.+ const redirect = !isSearchCrawler(ua);if (!agentId || !isUuid(agentId)) {- return passthrough(res, origin);+ return generic(res, origin, agentId, redirect);}let agent;try {[agent] = await sql`- SELECT i.id, i.name, i.description, i.skills+ SELECT i.id, i.name, i.description, i.skills, i.home_url,+ i.erc8004_agent_id, i.chain_id, i.created_at, i.updated_atFROM agent_identities i- WHERE i.id = ${agentId} AND i.deleted_at IS NULL+ WHERE i.id = ${agentId} AND i.deleted_at IS NULL AND i.is_public = trueLIMIT 1`;} catch {- return passthrough(res, origin);+ // A read failure is ours, not the URL's. 503 asks the crawler to come+ // back; a 404 or a redirect here would drop a live page from the index.+ return unavailable(res, origin);}- if (!agent) return passthrough(res, origin);+ if (!agent) {+ // Agents and avatars have separate uuid spaces and links cross over+ // (see resolveEntity in src/avatar-page.js). Send the crawler to the+ // path that actually holds the entity instead of reporting nothing.+ let avatar = null;+ try {+ [avatar] = await sql`+ SELECT id FROM avatars+ WHERE id = ${agentId} AND deleted_at IS NULL AND visibility = 'public' LIMIT 1+ `;+ } catch {+ return unavailable(res, origin);+ }+ if (avatar) {+ res.statusCode = 301;+ res.setHeader('location', `${origin}/avatars/${agentId}`);+ res.setHeader('cache-control', 'public, max-age=300');+ res.end();+ return;+ }+ return gone(res, origin);+ }const title = agent.name || 'Agent';- const baseDesc = agent.description || 'An AI agent on three.ws — with a body, a place, and an identity.';- const skills = agent.skills || [];+ const baseDesc =+ agent.description || 'An AI agent on three.ws, with a body, a place, and an identity.';+ const skills = Array.isArray(agent.skills) ? agent.skills : [];const skillSuffix = skills.length? ` Skills: ${skills.slice(0, 4).join(', ')}${skills.length > 4 ? '…' : ''}.`: '';const desc = baseDesc + skillSuffix;- // Unfurl with the dynamic wallet TRADING CARD (api/og/agent.js) — the same+ // Unfurl with the dynamic wallet TRADING CARD (api/og/agent.js) - the same// living card shown on the page (avatar, vanity address, live net worth, P&L,// reputation tier, finish), rendered from real data. The card embeds the// avatar itself, so a shared link previews with identity + wallet, not a bare// thumbnail. The card endpoint falls back to the brand image if it can't render.const ogImage = `${origin}/api/og/agent?id=${encodeURIComponent(agentId)}`;-const pageUrl = `${origin}/agents/${agentId}`;+ const facts = [];+ if (skills.length) {+ facts.push({+ term: 'Skills',+ detail: skills.slice(0, 12).map((s) => esc(s)).join(', '),+ });+ }+ if (agent.erc8004_agent_id) {+ facts.push({+ term: 'On-chain identity',+ detail: `ERC-8004 agent #${esc(agent.erc8004_agent_id)}${agent.chain_id ? ` on chain ${esc(agent.chain_id)}` : ''}`,+ });+ }+ if (agent.home_url && /^https?:\/\//i.test(agent.home_url)) {+ const h = esc(agent.home_url);+ facts.push({ term: 'Home', detail: `<a href="${h}" rel="nofollow noopener">${h}</a>` });+ }+ if (agent.created_at) facts.push({ term: 'Registered', detail: isoDay(agent.created_at) });+res.statusCode = 200;res.setHeader('content-type', 'text/html; charset=utf-8');res.setHeader('cache-control', 'public, max-age=60, s-maxage=600, stale-while-revalidate=3600');- res.end(renderHtml({ agentId, title, desc, pageUrl, ogImage, origin }));+ res.end(+ renderCrawlerPage({+ title,+ desc,+ pageUrl,+ ogImage,+ origin,+ ogType: 'profile',+ frameButton: `Meet ${title}`,+ redirect,+ // An untouched onboarding row is identical to every other one, so it+ // unfurls and renders but never asks to be indexed.+ noindex: !isIndexableAgent(agent),+ trail: [+ { name: 'Home', path: '/' },+ { name: 'Agents', path: '/agents' },+ { name: title, path: `/agents/${agentId}` },+ ],+ facts,+ // No skill chips: the directory filters on ?q= over name and+ // description, not on a skill facet, so a chip would look like a+ // filter and return an unfiltered list. The skills are in the facts+ // row and in featureList below.+ actions: [+ { label: 'View in AR', href: `/agents/${agentId}/ar`, primary: true },+ { label: 'Agent directory', href: '/agents' },+ { label: 'Build an agent', href: '/create' },+ ],+ related: [+ { label: 'Agent directory', href: '/agents' },+ { label: 'Agent economy', href: '/agent-economy' },+ { label: 'Agent marketplace', href: '/marketplace' },+ { label: 'Avatar gallery', href: '/gallery' },+ ],+ jsonLd: {+ '@type': 'SoftwareApplication',+ '@id': pageUrl,+ name: title,+ description: desc,+ url: pageUrl,+ image: ogImage,+ applicationCategory: 'AI agent',+ operatingSystem: 'Web',+ ...(skills.length ? { featureList: skills.slice(0, 12) } : {}),+ ...(agent.created_at ? { dateCreated: isoDay(agent.created_at) } : {}),+ ...(agent.updated_at ? { dateModified: isoDay(agent.updated_at) } : {}),+ publisher: { '@type': 'Organization', name: 'three.ws', url: origin },+ },+ }),+ );});-function passthrough(res, origin) {- res.statusCode = 302;- res.setHeader('location', `${origin}/agents`);- res.setHeader('cache-control', 'no-cache');- res.end();+/** An id we cannot read: keep the unfurl useful, keep the URL out of the index. */+function generic(res, origin, agentId, redirect) {+ const target = agentId ? `/agents/${encodeURIComponent(agentId)}` : '/agents';+ res.statusCode = 200;+ res.setHeader('content-type', 'text/html; charset=utf-8');+ res.setHeader('cache-control', 'public, max-age=60, s-maxage=600');+ res.end(+ renderCrawlerNotFound({+ heading: 'Agent on three.ws',+ message: 'Open this agent on three.ws to see its body, its wallet, and what it can do.',+ origin,+ actions: [+ { label: 'Open the agent', href: target },+ { label: 'Agent directory', href: '/agents' },+ ],+ redirect: redirect ? target : '',+ }),+ );}-function renderHtml({ agentId, title, desc, pageUrl, ogImage, origin }) {- const t = esc(title);- const d = esc(desc);- const agentUrl = `/agents/${agentId}`;- return `<!doctype html>-<html lang="en">-<head>- <meta charset="utf-8">- <meta http-equiv="X-UA-Compatible" content="IE=edge">- <title>${t} — three.ws</title>- <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover">- <meta name="description" content="${d}">- <meta name="theme-color" content="#06070a">-- <meta property="og:type" content="profile">- <meta property="og:site_name" content="three.ws">- <meta property="og:title" content="${t} — three.ws">- <meta property="og:description" content="${d}">- <meta property="og:url" content="${esc(pageUrl)}">- <meta property="og:image" content="${esc(ogImage)}">- <meta property="og:image:width" content="1200">- <meta property="og:image:height" content="630">- <meta property="og:image:alt" content="${t} on three.ws">-- <meta name="twitter:card" content="summary_large_image">- <meta name="twitter:title" content="${t} — three.ws">- <meta name="twitter:description" content="${d}">- <meta name="twitter:image" content="${esc(ogImage)}">- <meta name="twitter:creator" content="@trythreews">-- <meta property="fc:frame" content="vNext">- <meta property="fc:frame:image" content="${esc(ogImage)}">- <meta property="fc:frame:image:aspect_ratio" content="1.91:1">- <meta property="fc:frame:button:1" content="Meet ${t}">- <meta property="fc:frame:button:1:action" content="link">- <meta property="fc:frame:button:1:target" content="${esc(pageUrl)}">-- <link rel="canonical" href="${esc(pageUrl)}">- <link rel="shortcut icon" href="/favicon.ico">-- <style>- html,body{margin:0;padding:0;background:#06070a;color:#e0e0e0;font-family:Inter,system-ui,sans-serif;height:100%}- .shell{display:grid;place-items:center;min-height:100vh;text-align:center;padding:2rem;gap:1rem}- .shell a{color:#e0e0e0;text-decoration:underline;text-underline-offset:3px}- .spinner{width:28px;height:28px;border:2px solid rgba(255,255,255,0.1);border-top-color:rgba(255,255,255,0.6);border-radius:50%;animation:spin 0.9s linear infinite;margin:0 auto}- @keyframes spin{to{transform:rotate(360deg)}}- p{margin:0;color:rgba(255,255,255,0.5);font-size:14px}- </style>-</head>-<body>- <noscript>- <div class="shell">- <h1>${t}</h1>- <p>${d}</p>- <p><a href="${esc(pageUrl)}">View agent</a> · <a href="${esc(origin)}/agents">Browse agents</a></p>- </div>- </noscript>- <div class="shell" aria-live="polite">- <div class="spinner" aria-hidden="true"></div>- <p>Loading ${t}…</p>- </div>- <script>(function(){window.location.replace(${JSON.stringify(agentUrl)});})()</script>-</body>-</html>`;+/** Nothing public lives at this id. 404 so the URL leaves the index cleanly. */+function gone(res, origin) {+ res.statusCode = 404;+ res.setHeader('content-type', 'text/html; charset=utf-8');+ res.setHeader('cache-control', 'public, max-age=300');+ res.end(+ renderCrawlerNotFound({+ heading: 'This agent is not public',+ message:+ 'The agent behind this link was retired, or its owner keeps it private. The directory lists the ones that are live.',+ origin,+ actions: [+ { label: 'Agent directory', href: '/agents' },+ { label: 'Build an agent', href: '/create' },+ ],+ }),+ );}-function esc(s) {- return String(s ?? '').replace(/[&<>"']/g, (c) =>- ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[c],+/** Our read failed. Ask the crawler to retry rather than losing the page. */+function unavailable(res, origin) {+ res.statusCode = 503;+ res.setHeader('content-type', 'text/html; charset=utf-8');+ res.setHeader('retry-after', '120');+ res.setHeader('cache-control', 'no-store');+ res.end(+ renderCrawlerNotFound({+ heading: 'Agent temporarily unavailable',+ message: 'three.ws could not load this agent just now. Please try again in a moment.',+ origin,+ actions: [{ label: 'Agent directory', href: '/agents' }],+ }),);}++/** YYYY-MM-DD, the only precision a crawler page needs. */+function isoDay(v) {+ const t = v instanceof Date ? v : new Date(v);+ return Number.isNaN(t.getTime()) ? '' : t.toISOString().slice(0, 10);+}diff --git a/api/avatar-detail-og.js b/api/avatar-detail-og.jsindex ad38609b9..47e02099a 100644--- a/api/avatar-detail-og.js+++ b/api/avatar-detail-og.js@@ -1,40 +1,53 @@/**- * SSR OG page for avatar detail (/avatars/:id)- * --------------------------------------------+ * Crawler page for avatar detail (/avatars/:id)+ * ---------------------------------------------* GET /api/avatar-detail-og?id=<avatarId>*- * Wired via vercel.json: when a social crawler hits /avatars/<uuid>,- * a User-Agent "has" condition rewrites to this endpoint. Returns a minimal- * HTML page with OG + Twitter Card + Farcaster Frame meta so shared avatar- * links unfurl with the avatar's real name, description, and rendered card.+ * Wired via vercel.json: when a crawler User-Agent hits /avatars/<uuid>, a+ * "has" condition rewrites to this endpoint. Real browsers never reach it, so+ * this is the only version of the page a search engine or a link unfurler ever+ * sees. It therefore has to carry the avatar's real content, not just meta+ * tags: see the header of api/_lib/crawler-page.js for why a spinner plus a+ * <noscript> block collapsed 45k avatar URLs into a single duplicate cluster.*- * Real browsers never reach this route - the rewrite only fires for known- * bot User-Agents. Mirrors api/agent-detail-og.js.+ * Mirrors api/agent-detail-og.js.*/import { sql } from './_lib/db.js';import { cors, method, wrap } from './_lib/http.js';import { env } from './_lib/env.js';import { isUuid } from './_lib/validate.js';+import { esc, isSearchCrawler, renderCrawlerPage, renderCrawlerNotFound } from './_lib/crawler-page.js';+import { isIndexableAvatar } from './_lib/indexable-entity.js';export default wrap(async (req, res) => {if (cors(req, res, { methods: 'GET,OPTIONS' })) return;- // Crawler-only SSR read. Without this, a POST answered with the 302- // passthrough instead of the 405 the advertised Allow set promises.+ // Crawler-only SSR read. Without this, a POST answered with a page instead+ // of the 405 the advertised Allow set promises.if (!method(req, res, ['GET'])) return;const url = new URL(req.url, 'http://x');const avatarId = url.searchParams.get('id');const origin = env.APP_ORIGIN || 'https://three.ws';-+ const ua = req.headers?.['user-agent'];+ // Only a non-indexing scraper gets bounced to the app. An indexing crawler+ // runs that script, lands back on this same URL behind the same UA branch,+ // and loops on an identical response.+ const redirect = !isSearchCrawler(ua);++ // Demo and legacy ids are not uuids. They still resolve in the app, so keep+ // the bounce and a valid unfurl, but keep them out of the index rather than+ // inventing a page for an entity we cannot read.if (!avatarId || !isUuid(avatarId)) {- return passthrough(res, origin);+ return generic(res, origin, avatarId, redirect);}let avatar;try {[avatar] = await sql`- SELECT a.id, a.name, a.description, a.tags, a.model_category,+ SELECT a.id, a.name, a.description, a.alt_text, a.tags, a.model_category,+ a.view_count, a.fork_count, a.parent_avatar_id,+ a.created_at, a.updated_at,u.username AS owner_usernameFROM avatars aLEFT JOIN users u ON u.id = a.owner_id AND u.deleted_at IS NULL@@ -42,16 +55,43 @@ export default wrap(async (req, res) => {LIMIT 1`;} catch {- return passthrough(res, origin);+ // A read failure is ours, not the URL's. 503 asks the crawler to come+ // back; a 404 or a redirect here would drop a live page from the index.+ return unavailable(res, origin);}- if (!avatar) return passthrough(res, origin);+ if (!avatar) {+ // Agents and avatars have separate uuid spaces and links cross over+ // (see resolveEntity in src/avatar-page.js). Send the crawler to the+ // path that actually holds the entity instead of reporting nothing.+ let agent = null;+ try {+ [agent] = await sql`+ SELECT id FROM agent_identities+ WHERE id = ${avatarId} AND deleted_at IS NULL LIMIT 1+ `;+ } catch {+ return unavailable(res, origin);+ }+ if (agent) {+ res.statusCode = 301;+ res.setHeader('location', `${origin}/agents/${avatarId}`);+ res.setHeader('cache-control', 'public, max-age=300');+ res.end();+ return;+ }+ return gone(res, origin);+ }const title = avatar.name || 'Avatar';const baseDesc =- avatar.description || 'A 3D avatar on three.ws - rigged, animated, and ready to become an agent.';- const tags = avatar.tags || [];- const tagSuffix = tags.length ? ` Tags: ${tags.slice(0, 4).join(', ')}${tags.length > 4 ? '…' : ''}.` : '';+ avatar.description ||+ avatar.alt_text ||+ 'A 3D avatar on three.ws - rigged, animated, and ready to become an agent.';+ const tags = Array.isArray(avatar.tags) ? avatar.tags : [];+ const tagSuffix = tags.length+ ? ` Tags: ${tags.slice(0, 4).join(', ')}${tags.length > 4 ? '…' : ''}.`+ : '';const byline = avatar.owner_username ? ` By @${avatar.owner_username}.` : '';const desc = baseDesc + byline + tagSuffix;@@ -59,90 +99,148 @@ export default wrap(async (req, res) => {// turntable render with name plate, from real data. The card endpoint falls// back to the brand image if it can't render.const ogImage = `${origin}/api/avatars/${encodeURIComponent(avatarId)}/og`;-const pageUrl = `${origin}/avatars/${avatarId}`;+ const facts = [];+ if (avatar.owner_username) {+ const u = esc(avatar.owner_username);+ facts.push({ term: 'Creator', detail: `<a href="/u/${u}" rel="author">@${u}</a>` });+ }+ if (avatar.model_category) facts.push({ term: 'Category', detail: esc(avatar.model_category) });+ if (avatar.parent_avatar_id) {+ const parent = esc(avatar.parent_avatar_id);+ facts.push({ term: 'Remixed from', detail: `<a href="/avatars/${parent}">the original avatar</a>` });+ }+ if (Number(avatar.fork_count) > 0) {+ facts.push({ term: 'Remixes', detail: `${Number(avatar.fork_count)} on three.ws` });+ }+ if (Number(avatar.view_count) > 0) {+ facts.push({ term: 'Views', detail: String(Number(avatar.view_count)) });+ }+ if (avatar.created_at) facts.push({ term: 'Published', detail: isoDay(avatar.created_at) });+res.statusCode = 200;res.setHeader('content-type', 'text/html; charset=utf-8');res.setHeader('cache-control', 'public, max-age=60, s-maxage=600, stale-while-revalidate=3600');- res.end(renderHtml({ avatarId, title, desc, pageUrl, ogImage, origin }));+ res.end(+ renderCrawlerPage({+ title,+ desc,+ pageUrl,+ ogImage,+ origin,+ ogType: 'profile',+ frameButton: `Meet ${title}`,+ redirect,+ // An untouched onboarding row is identical to every other one, so it+ // unfurls and renders but never asks to be indexed.+ noindex: !isIndexableAvatar(avatar),+ trail: [+ { name: 'Home', path: '/' },+ { name: 'Gallery', path: '/gallery' },+ { name: title, path: `/avatars/${avatarId}` },+ ],+ facts,+ tags: tags.slice(0, 12).map((tag) => ({+ label: tag,+ href: `/gallery?tag=${encodeURIComponent(tag)}`,+ })),+ actions: [+ { label: 'View in AR', href: `/avatars/${avatarId}/ar`, primary: true },+ { label: 'Browse the gallery', href: '/gallery' },+ { label: 'Create your own', href: '/create' },+ ],+ related: [+ { label: 'Avatar gallery', href: '/gallery' },+ { label: 'Animation library', href: '/animations' },+ { label: 'Build an embeddable widget', href: '/studio' },+ { label: 'Agent directory', href: '/agents' },+ ],+ jsonLd: {+ '@type': '3DModel',+ '@id': pageUrl,+ name: title,+ description: desc,+ url: pageUrl,+ image: ogImage,+ encodingFormat: 'model/gltf-binary',+ isFamilyFriendly: true,+ ...(tags.length ? { keywords: tags.join(', ') } : {}),+ ...(avatar.created_at ? { dateCreated: isoDay(avatar.created_at) } : {}),+ ...(avatar.updated_at ? { dateModified: isoDay(avatar.updated_at) } : {}),+ ...(avatar.owner_username+ ? {+ creator: {+ '@type': 'Person',+ name: `@${avatar.owner_username}`,+ url: `${origin}/u/${avatar.owner_username}`,+ },+ }+ : {}),+ publisher: { '@type': 'Organization', name: 'three.ws', url: origin },+ },+ }),+ );});-function passthrough(res, origin) {- res.statusCode = 302;- res.setHeader('location', `${origin}/gallery`);- res.setHeader('cache-control', 'no-cache');- res.end();+/** An id we cannot read: keep the unfurl useful, keep the URL out of the index. */+function generic(res, origin, avatarId, redirect) {+ const target = avatarId ? `/avatars/${encodeURIComponent(avatarId)}` : '/gallery';+ res.statusCode = 200;+ res.setHeader('content-type', 'text/html; charset=utf-8');+ res.setHeader('cache-control', 'public, max-age=60, s-maxage=600');+ res.end(+ renderCrawlerNotFound({+ heading: 'Avatar on three.ws',+ message:+ 'Open this avatar in the three.ws studio to view it in 3D, animate it, and embed it anywhere.',+ origin,+ actions: [+ { label: 'Open the studio', href: target },+ { label: 'Browse the gallery', href: '/gallery' },+ ],+ redirect: redirect ? target : '',+ }),+ );}-function renderHtml({ avatarId, title, desc, pageUrl, ogImage, origin }) {- const t = esc(title);- const d = esc(desc);- const avatarUrl = `/avatars/${avatarId}`;- return `<!doctype html>-<html lang="en">-<head>- <meta charset="utf-8">- <meta http-equiv="X-UA-Compatible" content="IE=edge">- <title>${t} · three.ws</title>- <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover">- <meta name="description" content="${d}">- <meta name="theme-color" content="#06070a">-- <meta property="og:type" content="profile">- <meta property="og:site_name" content="three.ws">- <meta property="og:title" content="${t} · three.ws">- <meta property="og:description" content="${d}">- <meta property="og:url" content="${esc(pageUrl)}">- <meta property="og:image" content="${esc(ogImage)}">- <meta property="og:image:width" content="1200">- <meta property="og:image:height" content="630">- <meta property="og:image:alt" content="${t} on three.ws">-- <meta name="twitter:card" content="summary_large_image">- <meta name="twitter:title" content="${t} · three.ws">- <meta name="twitter:description" content="${d}">- <meta name="twitter:image" content="${esc(ogImage)}">- <meta name="twitter:creator" content="@trythreews">-- <meta property="fc:frame" content="vNext">- <meta property="fc:frame:image" content="${esc(ogImage)}">- <meta property="fc:frame:image:aspect_ratio" content="1.91:1">- <meta property="fc:frame:button:1" content="Meet ${t}">- <meta property="fc:frame:button:1:action" content="link">- <meta property="fc:frame:button:1:target" content="${esc(pageUrl)}">-- <link rel="canonical" href="${esc(pageUrl)}">- <link rel="shortcut icon" href="/favicon.ico">-- <style>- html,body{margin:0;padding:0;background:#06070a;color:#e0e0e0;font-family:Inter,system-ui,sans-serif;height:100%}- .shell{display:grid;place-items:center;min-height:100vh;text-align:center;padding:2rem;gap:1rem}- .shell a{color:#e0e0e0;text-decoration:underline;text-underline-offset:3px}- .spinner{width:28px;height:28px;border:2px solid rgba(255,255,255,0.1);border-top-color:rgba(255,255,255,0.6);border-radius:50%;animation:spin 0.9s linear infinite;margin:0 auto}- @keyframes spin{to{transform:rotate(360deg)}}- p{margin:0;color:rgba(255,255,255,0.5);font-size:14px}- </style>-</head>-<body>- <noscript>- <div class="shell">- <h1>${t}</h1>- <p>${d}</p>- <p><a href="${esc(pageUrl)}">View avatar</a> · <a href="${esc(origin)}/gallery">Browse the gallery</a></p>- </div>- </noscript>- <div class="shell" aria-live="polite">- <div class="spinner" aria-hidden="true"></div>- <p>Loading ${t}…</p>- </div>- <script>(function(){window.location.replace(${JSON.stringify(avatarUrl)});})()</script>-</body>-</html>`;+/** Nothing public lives at this id. 404 so the URL leaves the index cleanly. */+function gone(res, origin) {+ res.statusCode = 404;+ res.setHeader('content-type', 'text/html; charset=utf-8');+ res.setHeader('cache-control', 'public, max-age=300');+ res.end(+ renderCrawlerNotFound({+ heading: 'This avatar is not public',+ message:+ 'The avatar behind this link was removed, or its owner keeps it private. The gallery has thousands of others.',+ origin,+ actions: [+ { label: 'Browse the gallery', href: '/gallery' },+ { label: 'Create an avatar', href: '/create' },+ ],+ }),+ );}-function esc(s) {- return String(s ?? '').replace(/[&<>"']/g, (c) =>- ({ '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' })[c],+/** Our read failed. Ask the crawler to retry rather than losing the page. */+function unavailable(res, origin) {+ res.statusCode = 503;+ res.setHeader('content-type', 'text/html; charset=utf-8');+ res.setHeader('retry-after', '120');+ res.setHeader('cache-control', 'no-store');+ res.end(+ renderCrawlerNotFound({+ heading: 'Avatar temporarily unavailable',+ message: 'three.ws could not load this avatar just now. Please try again in a moment.',+ origin,+ actions: [{ label: 'Browse the gallery', href: '/gallery' }],+ }),);}++/** YYYY-MM-DD, the only precision a crawler page needs. */+function isoDay(v) {+ const t = v instanceof Date ? v : new Date(v);+ return Number.isNaN(t.getTime()) ? '' : t.toISOString().slice(0, 10);+}diff --git a/api/sitemap/[type].js b/api/sitemap/[type].jsindex 807838738..d7de42de0 100644--- a/api/sitemap/[type].js+++ b/api/sitemap/[type].js@@ -17,6 +17,7 @@ import { reportServerError, redactUrl } from '../_lib/http.js';import { getMonths, loadMonth } from '../_lib/news-archive-store.js';import { storyPath } from '../../src/shared/news-links.js';import { isSuppressed } from '../_lib/news-rights.js';+import { isIndexableAgent, isIndexableAvatar } from '../_lib/indexable-entity.js';const ORIGIN = env.APP_ORIGIN;const MAX_URLS = 45_000;@@ -155,35 +156,54 @@ async function coreSitemap() {}async function agentsSitemap() {+ // Drop the untouched onboarding rows: 1,359 of the 3,397 public agents still+ // carried the starter name and description on 2026-09-07, and they are+ // duplicates of each other by construction (api/_lib/indexable-entity.js).+ // The filter runs in JS rather than SQL so one definition of "indexable"+ // serves both this file and the crawler page, which must agree. The+ // predicate only tests whether the prose is empty and whether it opens with+ // the starter copy, so a prefix is all it needs and the row stays small.const rows = await sql`- select id, updated_at, created_at+ select id, name, left(description, 64) as description, updated_at, created_atfrom agent_identitieswhere deleted_at is null and is_public = trueorder by coalesce(updated_at, created_at) desclimit ${MAX_URLS}`;- return rows.map((r) => ({- loc: `${ORIGIN}/agents/${r.id}`,- lastmod: fmtDate(r.updated_at || r.created_at),- changefreq: 'weekly',- priority: '0.7',- }));+ return rows+ .filter(isIndexableAgent)+ .map((r) => ({+ loc: `${ORIGIN}/agents/${r.id}`,+ lastmod: fmtDate(r.updated_at || r.created_at),+ changefreq: 'weekly',+ priority: '0.7',+ }));}async function avatarsSitemap() {+ // Same filter as agentsSitemap: an avatar still carrying its default name with+ // nothing written about it cannot be told apart from the next one. It removes+ // far less here (170 of 67,164 on 2026-09-07) because a generated avatar+ // still gets its own description and tags. Filtering after the cap rather+ // than widening the fetch is deliberate: the public set is already half again+ // larger than the 45k ceiling, so the file is truncated either way and+ // pulling 90k rows into memory to reclaim 0.25% of the slots is not a trade.const rows = await sql`- select id, updated_at, created_at+ select id, name, left(description, 64) as description, left(alt_text, 64) as alt_text,+ updated_at, created_atfrom avatarswhere deleted_at is null and visibility = 'public'order by coalesce(updated_at, created_at) desclimit ${MAX_URLS}`;- return rows.map((r) => ({- loc: `${ORIGIN}/avatars/${r.id}`,- lastmod: fmtDate(r.updated_at || r.created_at),- changefreq: 'weekly',- priority: '0.6',- }));+ return rows+ .filter(isIndexableAvatar)+ .map((r) => ({+ loc: `${ORIGIN}/avatars/${r.id}`,+ lastmod: fmtDate(r.updated_at || r.created_at),+ changefreq: 'weekly',+ priority: '0.6',+ }));}async function widgetsSitemap() {diff --git a/docs/ops/README.md b/docs/ops/README.mdindex ec497f1c0..25f8eca6d 100644--- a/docs/ops/README.md+++ b/docs/ops/README.md@@ -30,6 +30,7 @@ Start with [gcp-production.md](gcp-production.md); it is the complete operationa| [runtime-flags.md](runtime-flags.md) | DB-backed feature flags that flip platform behavior without a redeploy. || [x402-discovery-listings.md](x402-discovery-listings.md) | Getting paid endpoints listed and ranked on x402scan, the Bazaar, and other directories. || [seo-keyword-plan.md](seo-keyword-plan.md) | Verified keyword landscape and content calendar (snapshot dated 2026-07-17). |+| [search-indexing.md](search-indexing.md) | How pages reach a search index: the three serving paths, the crawler-page duplicate cluster and its fix, robots.txt vs noindex, SPA soft 404s, and how to check a URL as a crawler. || [wallet-key-migration.md](wallet-key-migration.md) | Incident record: stranded pool-agent wallets after the WALLET_ENCRYPTION_KEY migration. Also the standing runbook for where production's credentials live in Secret Manager, how to publish a new version, and the owner-gated master-wallet rotation. || [stranded-wallets.md](stranded-wallets.md) | The standing owner decision on custodial wallets sealed by that rotation: measured list, why recovery is impossible, what crediting or writing off the customer balances costs, and the commands for each. || [forge-multiview-migration-handoff.md](forge-multiview-migration-handoff.md) | Historical hand-off for the forge multi-view migrations (June 2026); superseded by gcp-production.md. |diff --git a/docs/ops/search-indexing.md b/docs/ops/search-indexing.mdnew file mode 100644index 000000000..63a038f87--- /dev/null+++ b/docs/ops/search-indexing.md@@ -0,0 +1,234 @@+# Search indexing++How three.ws pages reach a search index, what broke, and what to check before+concluding a page "just isn't ranking yet". Companion to+[seo-keyword-plan.md](./seo-keyword-plan.md), which covers what to write;+this doc covers whether a crawler can read it at all.++---++## The three ways a page is served to a crawler++Every public URL falls into one of these, and the failure modes are different:++1. **A static page** (`pages/*.html`, most of `data/pages.json`). The HTML on+ disk is the HTML a crawler sees. `scripts/inject-seo-meta.mjs` backfills the+ title, description, canonical, OG/Twitter tags and JSON-LD from+ `data/pages.json`, so the meta a crawler reads matches the catalog. Run it+ with `--write` after adding a page; it never overwrites a tag a page already+ has.++2. **A server-rendered page** (`api/*-og.js`, `api/markets/*`). The handler+ composes the document per request from the database. `/markets/news/...` and+ the crawler pages below are here.++3. **A single-page-app shell.** One HTML file backs an unbounded URL space:+ `/coin/:id` renders `pages/coin.html`, `/m/:id` renders `pages/model.html`,+ and so on. The shell has no content and no canonical until the controller+ fills it in from an API call. Google indexes the *rendered* DOM, so this+ works, but only if the controller sets the head tags. `src/seo-meta.js`+ exists for exactly that (`setCanonical`, `markNoindex`).++`/avatars/:id` and `/agents/:id` are the interesting case: a browser gets the+SPA shell, and a crawler UA is rewritten by `vercel.json` to a server-rendered+page instead (`api/avatar-detail-og.js`, `api/agent-detail-og.js`). Those two+URL spaces are also the largest on the site: the avatars sub-sitemap alone+submits up to 45,000 URLs.++---++## The duplicate-cluster failure (2026-09)++Search Console, page indexing report, snapshot 2026-09-03: **30.2k indexed,+25.1k not indexed**, and the single largest exclusion was **18,768 pages under+"Duplicate without user-selected canonical"**, alongside 1,101 "Page with+redirect", 1,382 "Crawled - currently not indexed" and 813 "Duplicate, Google+chose different canonical than user".++The cause was in the crawler pages, and it is worth understanding because the+same shape is easy to reintroduce. The rendered page carried the avatar's name+and description in `<meta>` tags, and the visible document was:++```html+<noscript>+ <div class="shell"><h1>Shaw</h1><p>Rigged, walk-ready avatar…</p></div>+</noscript>+<div class="shell"><div class="spinner"></div><p>Loading Shaw…</p></div>+<script>(function(){window.location.replace("/avatars/19da5ca4-…");})()</script>+```++Two things go wrong, and they compound:++- **The redirect target is the URL the crawler is already on.** The rewrite is+ keyed on the User-Agent, not the path, so re-requesting the same URL behind+ the same UA returns a byte-identical response and the navigation repeats. A+ crawler that runs JS (Googlebot, bingbot and Applebot all render before+ indexing) never lands anywhere else, and may bucket the URL as a redirect.+- **Rendering removes `<noscript>`.** The only real text on the page lived+ inside it, so the document that reached the indexer was a spinner over the+ word "Loading". Every avatar URL rendered to the same nothing.++45,000 URLs with no distinguishing content is one duplicate cluster 45,000 wide.+The canonical tag in the head could not save it: a cluster where nothing+differs has no preferred version to point at.++### The fix++`api/_lib/crawler-page.js` now renders both crawler pages, and:++- **The content is in the body.** Name, description, the 1200x630 card as a real+ `<img>`, a definition list of facts (creator, category, rig state, fork count,+ publication date for an avatar; skills, ERC-8004 identity, home URL for an+ agent), tag chips linking into `/gallery` or `/agents`, and a footer of+ related surfaces. Plus `3DModel` / `SoftwareApplication` JSON-LD and a+ `BreadcrumbList`. No `<noscript>`, because nothing hides in it any more.+- **The self-redirect is emitted only for crawlers that never index.**+ `isSearchCrawler()` splits indexing crawlers (Googlebot, bingbot, Applebot,+ GPTBot, ClaudeBot, PerplexityBot, and the rest) from link unfurlers+ (Twitterbot, Slackbot, Embedly, Iframely). An unfurler still gets bounced to+ the app, which is the friendly outcome for a human arriving behind a scraper+ UA. A search engine gets content and no script.+- **A missing entity 404s** instead of 302-ing to `/gallery` or `/agents`. Those+ redirects were the "Page with redirect" and "Soft 404" rows: a crawler asked+ about a deleted avatar and was told, in effect, "here is a different page".+ Now it is told the truth, on a `noindex` document, and the URL leaves the+ index.+- **An id that names the other store 301s across.** Agent and avatar UUIDs are+ separate spaces and links cross over (`resolveEntity` in `src/avatar-page.js`+ does the same thing client-side), so `/agents/<an avatar id>` sends a real 301+ to `/avatars/<id>` rather than reporting nothing.+- **A database failure returns 503, not 404.** A read failure is ours, not the+ URL's; a 404 there would drop live pages out of the index.+- **Private agents are no longer served at all.** The agent query filtered only+ on `deleted_at`, so an agent with `is_public = false` had its name and+ description rendered to anything sending a bot UA. It now matches the+ sitemap's predicate (`is_public = true`).++### Content parity, not cloaking++Serving crawlers a different document than browsers is dynamic serving, which is+allowed as long as the content is equivalent. It is: the crawler page carries+the same name, description, imagery, tags, and creator the interactive page+shows. What it omits is the WebGL viewport a crawler cannot run. Keep it that+way. If a fact appears on the crawler page that a visitor cannot find on the+real one, that is the line.++---++## The other duplicate source: untouched onboarding rows++Rendering real content only helps if the entities differ. Measured against+production on 2026-09-07:++| | Public | Indexable | Dropped |+|---|---|---|---|+| Agents | 3,397 | 2,038 | 1,359 (40%) |+| Avatars | 67,164 | 66,994 | 170 |++**1,163 of the 3,397 public agents were still named "My First Agent"**, and+1,158 of those still carried the starter description word for word. Those pages+are not thin, they are the same page with a different UUID, and no server-side+rendering can separate them.++[`api/_lib/indexable-entity.js`](../../api/_lib/indexable-entity.js) decides.+An agent is indexable unless it keeps *both* its default name and its starter+description; an avatar is indexable unless it keeps a default name with no prose+at all (a generated avatar still gets its own description and tag set, which is+why so few are dropped). Both the sitemap builders and the crawler pages import+the predicate, so the file and the page can never disagree.++A non-indexable entity is not hidden: its page still renders, still unfurls with+its own card, and still links onward. It is served `noindex, follow` and left+out of the sitemap. The moment its owner types a name or a description it+becomes indexable again, with no migration and no backfill, which is why the+rule reads the row instead of storing a flag.++Re-measure before quoting these numbers:++```bash+node --input-type=module -e "+import { config } from 'dotenv'; config({ path: '.env.local', quiet: true });+const { sql } = await import('./api/_lib/db.js');+const { isIndexableAgent } = await import('./api/_lib/indexable-entity.js');+const rows = await sql\`select name, description from agent_identities+ where deleted_at is null and is_public = true\`;+console.log(rows.length, '->', rows.filter(isIndexableAgent).length);+process.exit(0);"+```++---++## Indexed, though blocked by robots.txt++25 URLs sat in this state, and it is always the same mistake:+`public/robots.txt` disallowed `/dashboard`, `/settings` and `/my-agents`, while+every page under them already carried `<meta name="robots" content="noindex">`.+A crawler that is not allowed to fetch the page never sees the noindex, so the+URL stays in the index as a bare link. The two directives cancel each other out.++Crawling is what removes them, so those three prefixes are now open in+robots.txt and the `noindex` does the work. `pages/dashboard-next/data-api.html`,+`developers.html` and `billing.html` were missing the tag and now carry it.++**Rule: pick one.** `Disallow` keeps a page out of the crawl; `noindex` keeps it+out of the index. Never both on the same URL.++---++## Soft 404s from SPA shells++A shell that renders "not found" over a `200` is a soft 404. The router cannot+know the entity is missing, but the controller can, so it calls `markNoindex()`+from [`src/seo-meta.js`](../../src/seo-meta.js) at the moment it renders the+empty state. Wired into `src/coin-page.js`, `src/model-page.js` and+`src/signal-detail.js`. Any new shell with a not-found branch should do the+same, and should call `setCanonical()` once it knows which entity it is showing.++---++## Checking a page yourself++Fetch as a crawler rather than trusting a browser view, because the UA branch+changes the response:++```bash+UA='Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/128.0.0.0 Safari/537.36'+curl -sS -A "$UA" https://three.ws/avatars/<id> | grep -E 'canonical|robots|<h1>'+```++What a healthy indexable page looks like:++- `200`, exactly one `<link rel="canonical">`, pointing at itself.+- `<meta name="robots">` absent or `index, follow`.+- A `<h1>` and body copy that differ from every sibling URL.+- No `location.replace` anywhere in the document.++Sub-sitemap sizes, which is the fastest way to see how much of each space is+being submitted:++```bash+for t in core agents avatars widgets profiles news; do+ echo "$t: $(curl -sS https://three.ws/sitemap/$t.xml | grep -c '<loc>')"+done+```++`npm run audit:web` sweeps every page in `data/pages.json` in a real browser+(see [page-audit.md](./page-audit.md)); it catches console errors, not indexing+state, so run both.++---++## What to expect after a fix lands++Search Console does not recount on deploy. Google has to recrawl each URL, and+at this volume that is weeks, not days. The order things move in:++1. "Page with redirect" and "Soft 404" fall first: they are decided by the+ response, which changes immediately.+2. "Duplicate without user-selected canonical" drains as pages are re-rendered+ and start to differ from one another.+3. Indexed count rises last.++Use the URL Inspection tool on a handful of representative URLs to confirm the+rendered HTML is what this doc describes before reading anything into the+aggregate charts.diff --git a/docs/ops/seo-keyword-plan.md b/docs/ops/seo-keyword-plan.mdindex 1178030d9..53f3bfa9c 100644--- a/docs/ops/seo-keyword-plan.md+++ b/docs/ops/seo-keyword-plan.md@@ -2,7 +2,7 @@Verified keyword landscape and content calendar for three.ws across its product clusters: 3D generation, AI agents/MCP, Solana/x402/crypto, and enterprise AI (watsonx/Granite). Built 2026-07-17 from a deep-research pass (24 sources fetched, 25 claims adversarially verified, 23 confirmed) plus a live audit of the IBM Community publishing surface. Every SERP observation is dated 2026-07-17; re-verify before acting on this doc more than a quarter later.-Related: [page-audit.md](./page-audit.md) covers on-site quality; this doc covers what to write and where to publish it.+Related: [page-audit.md](./page-audit.md) covers on-site quality and [search-indexing.md](./search-indexing.md) covers whether a crawler can read a page at all; this doc covers what to write and where to publish it.---diff --git a/pages/agent-mind.html b/pages/agent-mind.htmlindex e1d5fdd26..639eb5644 100644--- a/pages/agent-mind.html+++ b/pages/agent-mind.html@@ -6,6 +6,10 @@<title data-i18n="agent_mind.meta_title">Mind Palace — three.ws</title><meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="theme-color" content="#06080f" />+ <!-- One agent, one indexable page: /agents/:id. This view is reachable at+ both /agents/:id/mind and /agent/:id/mind and renders the same shell for+ every agent, so indexing it would file thousands of identical URLs. -->+ <meta name="robots" content="noindex, follow" /><meta name="description" content="Walk through your agent's mind — every memory as a tangible object you can see, search, pin, and forget." data-i18n-attr="content:agent_mind.meta_description"/><meta property="og:type" content="website">diff --git a/pages/dashboard-next/billing.html b/pages/dashboard-next/billing.htmlindex 585f83e0f..0b1562833 100644--- a/pages/dashboard-next/billing.html+++ b/pages/dashboard-next/billing.html@@ -3,6 +3,7 @@<head><meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />+ <meta name="robots" content="noindex" /><title data-i18n="dashboard_next_billing.meta_title">Billing & Passes · Dashboard · three.ws</title><meta name="description" content="Manage your three.ws subscriptions and paid services: Premium pass status, on-chain Solana payment history, API keys minted by your passes, and the creator subscriptions you hold." data-i18n-attr="content:dashboard_next_billing.meta_description"/><link rel="icon" href="/favicon.ico" />diff --git a/pages/dashboard-next/data-api.html b/pages/dashboard-next/data-api.htmlindex ad416ea0a..a49f03899 100644--- a/pages/dashboard-next/data-api.html+++ b/pages/dashboard-next/data-api.html@@ -3,6 +3,7 @@<head><meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />+ <meta name="robots" content="noindex" /><title data-i18n="dashboard_next_data_api.meta_title">Data API · Dashboard · three.ws</title><meta name="description" content="Developer console for the three.ws Data API: buy the Premium pass with $THREE, SOL, or USDC, manage your API key, watch usage, and copy working quickstarts." data-i18n-attr="content:dashboard_next_data_api.meta_description"/><link rel="icon" href="/favicon.ico" />diff --git a/pages/dashboard-next/developers.html b/pages/dashboard-next/developers.htmlindex 0c612933f..fce7a2935 100644--- a/pages/dashboard-next/developers.html+++ b/pages/dashboard-next/developers.html@@ -3,6 +3,7 @@<head><meta charset="utf-8" /><meta name="viewport" content="width=device-width, initial-scale=1" />+ <meta name="robots" content="noindex" /><title data-i18n="dashboard_next_developers.meta_title">Developer Hub · Dashboard · three.ws</title><meta name="description" content="Avatar Render API, webhooks, SDKs, and integration tools for developers building with three.ws avatars." data-i18n-attr="content:dashboard_next_developers.meta_description"/><link rel="icon" href="/favicon.ico" />diff --git a/public/changelog.json b/public/changelog.jsonindex eb5eedcb5..64970f091 100644--- a/public/changelog.json+++ b/public/changelog.json@@ -1,5 +1,5 @@{- "generated_at": "2026-09-07T05:35:32.758Z",+ "generated_at": "2026-09-07T05:42:31.482Z","generated_by": "scripts/build-page-index.mjs from data/pages.json + data/changelog.json","site": {"name": "three.ws",diff --git a/public/features.json b/public/features.jsonindex 444ea7306..1228305d3 100644--- a/public/features.json+++ b/public/features.json@@ -1,5 +1,5 @@{- "generated_at": "2026-09-07T05:35:32.744Z",+ "generated_at": "2026-09-07T05:42:31.471Z","generated_by": "scripts/build-page-index.mjs from data/pages.json","site": {"name": "three.ws",diff --git a/public/robots.txt b/public/robots.txtindex f2c0840d8..4f3bc6336 100644--- a/public/robots.txt+++ b/public/robots.txt@@ -1,11 +1,15 @@User-agent: *Allow: /-Disallow: /my-agents-Disallow: /dashboard-Disallow: /settingsDisallow: /api/Disallow: /studio-deposit-harness*+# /dashboard, /settings and /my-agents are signed-in surfaces and every one of+# their pages carries <meta name="robots" content="noindex">. A Disallow here+# would hide that tag from the crawler that needs to read it: Google kept them+# in the index as bare URLs ("Indexed, though blocked by robots.txt") precisely+# because it was never allowed to fetch the page and see the noindex. Crawling+# is what removes them, so leave these open.+# Embed routes serve raw runtime, not human content; keep them out of SERPs.Disallow: /agent/*/embedDisallow: /widget/*/embeddiff --git a/src/coin-page.js b/src/coin-page.jsindex 9e0f7e70c..a9008f14e 100644--- a/src/coin-page.js+++ b/src/coin-page.js@@ -18,6 +18,7 @@ import {timeAgo,escapeHtml as esc,} from './shared/coin-format.js';+import { markNoindex } from './seo-meta.js';const TIME_RANGES = [{ label: '24H', days: 1 },@@ -1046,6 +1047,8 @@ async function loadMarkets(coin) {// ── Not found / error states ────────────────────────────────────────────────function renderNotFound(id) {+ // A 200 with "not found" in the body is a soft 404 to a crawler. Say so.+ markNoindex();const mintish = MINT_RE.test(id);$('cv-head').innerHTML = `<h1 class="cv-h1">Coin not found</h1>diff --git a/src/model-page.js b/src/model-page.jsindex 1a75047c6..25a78be46 100644--- a/src/model-page.js+++ b/src/model-page.js@@ -24,6 +24,7 @@ import {embedSnippet,chipsFor,} from './model-lib.js';+import { markNoindex, setCanonical } from './seo-meta.js';const log = createLogger('model-page');const $ = (id) => document.getElementById(id);@@ -131,8 +132,7 @@ function patchMeta() {set('meta[property="og:url"]', `https://three.ws/m/${c.id}`);set('meta[property="og:image"]', `https://three.ws/api/forge/${c.id}/og`);set('meta[name="twitter:image"]', `https://three.ws/api/forge/${c.id}/og`);- const canonical = document.querySelector('link[rel="canonical"]');- if (canonical) canonical.setAttribute('href', `https://three.ws/m/${c.id}`);+ setCanonical(`/m/${c.id}`);}// ── viewer hero ──────────────────────────────────────────────────────────────@@ -764,6 +764,8 @@ function renderSuggested() {// ── error states ─────────────────────────────────────────────────────────────function renderMissing(msg) {+ // A 200 with "not found" in the body is a soft 404 to a crawler. Say so.+ markNoindex();const shell = $('mp-shell');if (!shell) return;shell.setAttribute('aria-busy', 'false');diff --git a/src/seo-meta.js b/src/seo-meta.jsnew file mode 100644index 000000000..30cbfcba1--- /dev/null+++ b/src/seo-meta.js@@ -0,0 +1,51 @@+// Crawler-facing <head> state for pages the router builds from a template.+//+// One HTML file backs an unbounded URL space on this site: /coin/:id, /m/:id,+// /signals/:id and friends all render the same shell and fill it in from an API+// call. Two things follow, and neither is optional if the page should rank.+//+// 1. A canonical baked into the file would name the template, not the page,+// so the shell ships without one and the controller sets a self-referential+// canonical once it knows which entity it is showing. Google reads the+// rendered DOM, so a canonical written here counts.+//+// 2. When the entity does not exist, the shell still answers 200 with an+// apology in the body: a soft 404. The router cannot know, but the+// controller does, so it marks the page noindex the moment it renders that+// state. The URL then leaves the index as "excluded by noindex" instead of+// accumulating as "soft 404".++const ORIGIN = 'https://three.ws';++/**+ * Point this page's canonical at `path` (default: the URL being viewed, minus+ * query and hash). Creates the tag when the shell shipped without one.+ *+ * @param {string} [path] absolute path, leading slash included+ */+export function setCanonical(path = location.pathname) {+ let link = document.querySelector('link[rel="canonical"]');+ if (!link) {+ link = document.createElement('link');+ link.rel = 'canonical';+ document.head.appendChild(link);+ }+ link.href = `${ORIGIN}${path}`;+}++/**+ * Declare this URL unindexable, for a view that renders "not found" over a 200.+ * `follow` is kept so the links out of the empty state still carry the crawler+ * somewhere useful. Also drops any canonical the page set earlier: a canonical+ * on a noindex page only muddies which URL the directive applies to.+ */+export function markNoindex() {+ let meta = document.querySelector('meta[name="robots"]');+ if (!meta) {+ meta = document.createElement('meta');+ meta.name = 'robots';+ document.head.appendChild(meta);+ }+ meta.setAttribute('content', 'noindex, follow');+ document.querySelector('link[rel="canonical"]')?.remove();+}diff --git a/src/signal-detail.js b/src/signal-detail.jsindex 266dd01d1..c97b8e276 100644--- a/src/signal-detail.js+++ b/src/signal-detail.js@@ -10,6 +10,7 @@import { apiFetch } from './api.js';import { escapeHtml, fmtPct, compact, identicon } from './trader-format.js';+import { markNoindex } from './seo-meta.js';const $ = (s, r = document) => r.querySelector(s);@@ -257,6 +258,8 @@ function wireSubscribe() {}function errorState(msg) {+ // A 200 with "not found" in the body is a soft 404 to a crawler. Say so.+ markNoindex();const root = $('#sd-root');root.setAttribute('aria-busy', 'false');root.innerHTML = `<div class="sm-empty"><h1 id="sd-title" class="sd-error-title">${escapeHtml(msg)}</h1><p>This feed may have been paused, made unlisted, or never existed.</p><div class="sm-empty-actions"><a class="sm-cta" href="/signals">Browse all feeds →</a></div></div>`;diff --git a/tests/api/agent-detail-og.test.js b/tests/api/agent-detail-og.test.jsnew file mode 100644index 000000000..e1258ef1b--- /dev/null+++ b/tests/api/agent-detail-og.test.js@@ -0,0 +1,137 @@+// The bot-UA rewrite for /agents/:id lands crawlers on api/agent-detail-og.js.+// That page is the only version of the agent a search engine ever sees, so it+// carries the agent's real content, keeps private agents out entirely (the+// query used to omit is_public and served their name and description to+// anything sending a bot UA), and never hands an indexing crawler a script that+// navigates back to the URL it is already on.++import { describe, it, expect, beforeEach, vi } from 'vitest';+import { makeReq, makeRes } from '../_helpers/monetization.js';++const sqlState = { rows: [], queue: null, calls: 0, seen: [] };++vi.mock('../../api/_lib/db.js', () => {+ const sql = vi.fn(async (strings) => {+ sqlState.seen.push(strings.join(' ? '));+ const rows = sqlState.queue ? (sqlState.queue[sqlState.calls] ?? []) : sqlState.rows;+ sqlState.calls += 1;+ return rows;+ });+ return { sql, isDbUnavailableError: () => false, isDbCapacityError: () => false };+});++const { default: handler } = await import('../../api/agent-detail-og.js');++const AGENT_ID = '00000000-0000-4000-8000-0000000000a1';+const GOOGLEBOT =+ 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/128.0.0.0 Safari/537.36';+const UNFURLER = 'Mozilla/5.0 (compatible; TelegramBot; like TwitterBot)';++const AGENT_ROW = {+ id: AGENT_ID,+ name: 'Atlas',+ description: 'A market-watching agent.',+ skills: ['research', 'trading'],+ home_url: 'https://example.com/atlas',+ erc8004_agent_id: '4821',+ chain_id: 8453,+ created_at: new Date('2026-05-04T00:00:00Z'),+ updated_at: new Date('2026-06-01T00:00:00Z'),+};++async function invokeHtml(reqOpts) {+ const req = makeReq({ headers: { 'user-agent': GOOGLEBOT }, ...reqOpts });+ const res = makeRes();+ await handler(req, res);+ return { status: res.statusCode, headers: res.headers, html: res.body };+}++beforeEach(() => {+ sqlState.rows = [];+ sqlState.queue = null;+ sqlState.calls = 0;+ sqlState.seen = [];+});++describe('agent-detail-og', () => {+ it('rejects a non-GET method with 405 before touching the db', async () => {+ const { status } = await invokeHtml({+ method: 'POST',+ url: `/api/agent-detail-og?id=${AGENT_ID}`,+ });+ expect(status).toBe(405);+ expect(sqlState.calls).toBe(0);+ });++ it('only reads public agents', async () => {+ sqlState.rows = [AGENT_ROW];+ await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });+ expect(sqlState.seen[0]).toMatch(/is_public = true/);+ });++ it('404s an unknown agent instead of redirecting to the directory', async () => {+ const { status, html } = await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });+ expect(status).toBe(404);+ expect(html).toMatch(/name="robots" content="noindex/);+ expect(sqlState.calls).toBe(2);+ });++ it('301s to /avatars/:id when the id names an avatar, not an agent', async () => {+ sqlState.queue = [[], [{ id: AGENT_ID }]];+ const { status, headers } = await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });+ expect(status).toBe(301);+ expect(headers.location).toBe(`https://three.ws/avatars/${AGENT_ID}`);+ });++ it('renders the agent as real indexable content', async () => {+ sqlState.rows = [AGENT_ROW];+ const { status, html } = await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });+ expect(status).toBe(200);+ expect(html).toMatch(/<h1>Atlas<\/h1>/);+ expect(html).toContain('A market-watching agent.');+ expect(html).toContain('research, trading');+ expect(html).toContain('ERC-8004 agent #4821');+ expect(html).toMatch(/rel="canonical" href="https:\/\/three\.ws\/agents\//);+ expect(html).toContain('"@type":"SoftwareApplication"');+ expect(html).not.toContain('<noscript>');+ });++ it('marks the agent home link nofollow so a crawler does not pass authority out', async () => {+ sqlState.rows = [AGENT_ROW];+ const { html } = await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });+ expect(html).toMatch(/rel="nofollow noopener"/);+ });++ it('renders an untouched onboarding agent noindex, with no canonical', async () => {+ sqlState.rows = [+ {+ ...AGENT_ROW,+ name: 'My First Agent',+ description:+ 'A friendly starter agent. Edit the personality and attach a 3D avatar \u2014 it goes live immediately.',+ },+ ];+ const { status, html } = await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });+ // The page still renders and still unfurls; it just does not ask to rank.+ expect(status).toBe(200);+ expect(html).toMatch(/<h1>My First Agent<\/h1>/);+ expect(html).toContain('og:image');+ expect(html).toMatch(/name="robots" content="noindex, follow"/);+ expect(html).not.toContain('rel="canonical"');+ });++ it('never emits a self-navigating script for an indexing crawler', async () => {+ sqlState.rows = [AGENT_ROW];+ const { html } = await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });+ expect(html).not.toContain('location.replace');+ });++ it('still bounces a link unfurler to the app', async () => {+ sqlState.rows = [AGENT_ROW];+ const { html } = await invokeHtml({+ url: `/api/agent-detail-og?id=${AGENT_ID}`,+ headers: { 'user-agent': UNFURLER },+ });+ expect(html).toContain(`location.replace("/agents/${AGENT_ID}")`);+ });+});diff --git a/tests/api/avatar-detail-og.test.js b/tests/api/avatar-detail-og.test.jsindex d23d72759..d9e35e4bc 100644--- a/tests/api/avatar-detail-og.test.js+++ b/tests/api/avatar-detail-og.test.js@@ -1,18 +1,21 @@-// /avatars/:id serves a static shell, so social crawlers used to unfurl every-// shared avatar link with the generic site card. The bot-UA rewrite now lands-// them on api/avatar-detail-og.js, which must render the avatar's real name,-// description, and rendered-image URL - and must never leak anything for a-// private or missing avatar (302 passthrough instead).+// /avatars/:id serves a static shell, so a bot-UA rewrite lands crawlers on+// api/avatar-detail-og.js instead. That page is the only version of the avatar+// a search engine ever sees, so it has to carry the avatar's real content, must+// never leak a private or deleted avatar, and must never hand an indexing+// crawler a script that navigates back to the URL it is already on (that loop,+// plus content that lived only inside <noscript>, is what collapsed 45k avatar+// URLs into one duplicate cluster in Search Console).import { describe, it, expect, beforeEach, vi } from 'vitest';import { makeReq, makeRes } from '../_helpers/monetization.js';-const sqlState = { rows: [], calls: 0 };+const sqlState = { rows: [], queue: null, calls: 0 };vi.mock('../../api/_lib/db.js', () => {const sql = vi.fn(async () => {+ const rows = sqlState.queue ? (sqlState.queue[sqlState.calls] ?? []) : sqlState.rows;sqlState.calls += 1;- return sqlState.rows;+ return rows;});return { sql, isDbUnavailableError: () => false, isDbCapacityError: () => false };});@@ -20,9 +23,27 @@ vi.mock('../../api/_lib/db.js', () => {const { default: handler } = await import('../../api/avatar-detail-og.js');const AVATAR_ID = '00000000-0000-4000-8000-000000000042';+const GOOGLEBOT =+ 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/128.0.0.0 Safari/537.36';+const UNFURLER = 'Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)';++const AVATAR_ROW = {+ id: AVATAR_ID,+ name: 'Nova <Scout>',+ description: 'A chrome scout.',+ alt_text: null,+ tags: ['scout', 'chrome'],+ model_category: 'avatar',+ view_count: 41,+ fork_count: 3,+ parent_avatar_id: null,+ created_at: new Date('2026-05-04T00:00:00Z'),+ updated_at: new Date('2026-06-01T00:00:00Z'),+ owner_username: 'nirholas',+};async function invokeHtml(reqOpts) {- const req = makeReq(reqOpts);+ const req = makeReq({ headers: { 'user-agent': GOOGLEBOT }, ...reqOpts });const res = makeRes();await handler(req, res);return { status: res.statusCode, headers: res.headers, html: res.body };@@ -30,6 +51,7 @@ async function invokeHtml(reqOpts) {beforeEach(() => {sqlState.rows = [];+ sqlState.queue = null;sqlState.calls = 0;});@@ -43,48 +65,78 @@ describe('avatar-detail-og', () => {expect(sqlState.calls).toBe(0);});- it('302-passthroughs a malformed id without querying', async () => {- const { status, headers } = await invokeHtml({- url: '/api/avatar-detail-og?id=not-a-uuid',- });- expect(status).toBe(302);- expect(headers.location).toMatch(/\/gallery$/);+ it('serves a noindex page for a malformed id without querying', async () => {+ const { status, html } = await invokeHtml({ url: '/api/avatar-detail-og?id=not-a-uuid' });+ expect(status).toBe(200);+ expect(html).toMatch(/name="robots" content="noindex/);expect(sqlState.calls).toBe(0);});- it('302-passthroughs an unknown avatar', async () => {- const { status, headers } = await invokeHtml({- url: `/api/avatar-detail-og?id=${AVATAR_ID}`,- });- expect(status).toBe(302);- expect(headers.location).toMatch(/\/gallery$/);- expect(sqlState.calls).toBe(1);+ it('404s an unknown avatar instead of redirecting to the gallery', async () => {+ const { status, html } = await invokeHtml({ url: `/api/avatar-detail-og?id=${AVATAR_ID}` });+ expect(status).toBe(404);+ expect(html).toMatch(/name="robots" content="noindex/);+ // Avatar miss, then the cross-store probe for an agent with the same id.+ expect(sqlState.calls).toBe(2);});- it('renders OG meta from the avatar record for a public avatar', async () => {- sqlState.rows = [- {- id: AVATAR_ID,- name: 'Nova <Scout>',- description: 'A chrome scout.',- tags: ['scout', 'chrome'],- model_category: 'avatar',- owner_username: 'nirholas',- },- ];+ it('301s to /agents/:id when the id names an agent, not an avatar', async () => {+ sqlState.queue = [[], [{ id: AVATAR_ID }]];+ const { status, headers } = await invokeHtml({ url: `/api/avatar-detail-og?id=${AVATAR_ID}` });+ expect(status).toBe(301);+ expect(headers.location).toBe(`https://three.ws/agents/${AVATAR_ID}`);+ });++ it('renders the avatar as real indexable content, not a spinner', async () => {+ sqlState.rows = [AVATAR_ROW];const { status, headers, html } = await invokeHtml({url: `/api/avatar-detail-og?id=${AVATAR_ID}`,});expect(status).toBe(200);expect(headers['content-type']).toContain('text/html');- // Name is escaped, description + byline + tags flow into og:description.- expect(html).toContain('Nova <Scout>');+ // The name and description are in the body, not only in <meta>/<noscript>.+ expect(html).toMatch(/<h1>Nova <Scout><\/h1>/);expect(html).toContain('A chrome scout.');expect(html).toContain('@nirholas');expect(html).toContain(`/api/avatars/${AVATAR_ID}/og`);- expect(html).toContain(`/avatars/${AVATAR_ID}`);+ expect(html).toMatch(/rel="canonical" href="https:\/\/three\.ws\/avatars\//);expect(html).toContain('og:image');expect(html).toContain('twitter:card');- expect(html).toMatch(/rel="canonical"/);+ expect(html).not.toContain('<noscript>');+ // Structured data and internal links a crawler can follow.+ expect(html).toContain('application/ld+json');+ expect(html).toContain('"@type":"3DModel"');+ expect(html).toContain('BreadcrumbList');+ expect(html).toContain('href="/gallery"');+ });++ it('renders a default-named, undescribed avatar noindex', async () => {+ sqlState.rows = [{ ...AVATAR_ROW, name: 'Avatar', description: null, alt_text: null }];+ const { status, html } = await invokeHtml({ url: `/api/avatar-detail-og?id=${AVATAR_ID}` });+ expect(status).toBe(200);+ expect(html).toMatch(/name="robots" content="noindex, follow"/);+ expect(html).not.toContain('rel="canonical"');+ });++ it('never emits a self-navigating script for an indexing crawler', async () => {+ sqlState.rows = [AVATAR_ROW];+ const { html } = await invokeHtml({ url: `/api/avatar-detail-og?id=${AVATAR_ID}` });+ expect(html).not.toContain('location.replace');+ });++ it('still bounces a link unfurler to the app', async () => {+ sqlState.rows = [AVATAR_ROW];+ const { html } = await invokeHtml({+ url: `/api/avatar-detail-og?id=${AVATAR_ID}`,+ headers: { 'user-agent': UNFURLER },+ });+ expect(html).toContain(`location.replace("/avatars/${AVATAR_ID}")`);+ });++ it('escapes the avatar name everywhere it lands in the document', async () => {+ sqlState.rows = [{ ...AVATAR_ROW, name: '"><script>alert(1)</script>' }];+ const { html } = await invokeHtml({ url: `/api/avatar-detail-og?id=${AVATAR_ID}` });+ expect(html).not.toContain('<script>alert(1)</script>');+ expect(html).toContain('<script>alert(1)</script>');});});diff --git a/tests/api/crawler-page.test.js b/tests/api/crawler-page.test.jsnew file mode 100644index 000000000..54db98aa0--- /dev/null+++ b/tests/api/crawler-page.test.js@@ -0,0 +1,102 @@+// The UA split in api/_lib/crawler-page.js decides whether a crawler page ends+// with a script that navigates to the URL it is already on. Get it wrong in one+// direction and an indexing crawler loops on a content-free response; get it+// wrong in the other and a human who arrives behind a scraper UA is stranded on+// a static page. Both halves are asserted here.++import { describe, it, expect } from 'vitest';+import { isSearchCrawler, renderCrawlerPage, renderCrawlerNotFound } from '../../api/_lib/crawler-page.js';++const SEARCH = [+ 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/128.0.0.0 Safari/537.36',+ 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',+ 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',+ 'Mozilla/5.0 (Device; OS) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Safari/600.0 (Applebot/0.1)',+ 'Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)',+ 'Mozilla/5.0 (compatible; ClaudeBot/1.0; [email protected])',+ 'Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)',+ 'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)',+ 'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)',+];++const UNFURLERS = [+ 'Twitterbot/1.0',+ 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',+ 'Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)',+ 'Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)',+ 'TelegramBot (like TwitterBot)',+ 'Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)',+ 'Iframely/1.3.1 (+https://iframely.com/docs/about)',+ 'WhatsApp/2.19.81 A',+];++const PAGE = {+ title: 'Nova',+ desc: 'A chrome scout.',+ pageUrl: 'https://three.ws/avatars/abc',+ ogImage: 'https://three.ws/api/avatars/abc/og',+ origin: 'https://three.ws',+ trail: [+ { name: 'Home', path: '/' },+ { name: 'Gallery', path: '/gallery' },+ { name: 'Nova', path: '/avatars/abc' },+ ],+};++describe('isSearchCrawler', () => {+ it('recognises every indexing crawler we serve pages to', () => {+ for (const ua of SEARCH) expect(isSearchCrawler(ua), ua).toBe(true);+ });++ it('does not claim link unfurlers index anything', () => {+ for (const ua of UNFURLERS) expect(isSearchCrawler(ua), ua).toBe(false);+ });++ it('treats a missing user-agent as a non-indexer', () => {+ expect(isSearchCrawler(undefined)).toBe(false);+ expect(isSearchCrawler('')).toBe(false);+ expect(isSearchCrawler(null)).toBe(false);+ });+});++describe('renderCrawlerPage', () => {+ it('emits the self-navigating script only when asked', () => {+ expect(renderCrawlerPage({ ...PAGE, redirect: false })).not.toContain('location.replace');+ expect(renderCrawlerPage({ ...PAGE, redirect: true })).toContain(+ 'location.replace("/avatars/abc")',+ );+ });++ it('puts the content in the body and the canonical in the head', () => {+ const html = renderCrawlerPage(PAGE);+ expect(html).toMatch(/<h1>Nova<\/h1>/);+ expect(html).toContain('A chrome scout.');+ expect(html).toContain('<link rel="canonical" href="https://three.ws/avatars/abc">');+ expect(html).toMatch(/name="robots" content="index, follow/);+ expect(html).toContain('BreadcrumbList');+ });++ it('closes the ld+json script against a name that carries markup', () => {+ const html = renderCrawlerPage({+ ...PAGE,+ title: 'x</script><script>alert(1)</script>',+ jsonLd: { '@type': '3DModel', name: 'x</script><script>alert(1)</script>' },+ });+ expect(html).not.toContain('</script><script>alert(1)');+ expect(html).toContain('\\u003c/script');+ });+});++describe('renderCrawlerNotFound', () => {+ it('is always noindex and carries a way out', () => {+ const html = renderCrawlerNotFound({+ heading: 'Gone',+ message: 'Nothing here.',+ origin: 'https://three.ws',+ actions: [{ label: 'Gallery', href: '/gallery' }],+ });+ expect(html).toMatch(/name="robots" content="noindex, follow"/);+ expect(html).toContain('href="/gallery"');+ expect(html).not.toContain('rel="canonical"');+ });+});diff --git a/tests/api/indexable-entity.test.js b/tests/api/indexable-entity.test.jsnew file mode 100644index 000000000..c0a06bf14--- /dev/null+++ b/tests/api/indexable-entity.test.js@@ -0,0 +1,64 @@+// A third of the public agents on three.ws are onboarding rows nobody edited:+// same name, same description, different uuid. They are duplicates of each+// other by construction, so they stay out of the sitemap and their crawler page+// is served noindex. This is the predicate that decides, and it is shared by+// both so the two can never disagree.++import { describe, it, expect } from 'vitest';+import { isIndexableAgent, isIndexableAvatar } from '../../api/_lib/indexable-entity.js';++// The exact copy onboarding writes, escaped so the dash glyph the database+// stores does not have to be typed into this repo.+const STARTER =+ 'A friendly starter agent. Edit the personality and attach a 3D avatar \u2014 it goes live immediately.';++describe('isIndexableAgent', () => {+ it('rejects the untouched onboarding row', () => {+ expect(isIndexableAgent({ name: 'My First Agent', description: STARTER })).toBe(false);+ expect(isIndexableAgent({ name: 'my first agent', description: STARTER })).toBe(false);+ expect(isIndexableAgent({ name: ' My First Agent ', description: STARTER })).toBe(false);+ expect(isIndexableAgent({ name: 'My First Agent', description: null })).toBe(false);+ expect(isIndexableAgent({ name: 'Untitled Agent', description: '' })).toBe(false);+ });++ it('accepts a row where the owner supplied either half', () => {+ expect(isIndexableAgent({ name: 'Atlas', description: STARTER })).toBe(true);+ expect(isIndexableAgent({ name: 'My First Agent', description: 'Reads Solana mempools.' })).toBe(+ true,+ );+ expect(isIndexableAgent({ name: 'Atlas', description: 'Reads Solana mempools.' })).toBe(true);+ });++ it('rejects a row with no name at all', () => {+ expect(isIndexableAgent({ name: '', description: 'Something real.' })).toBe(false);+ expect(isIndexableAgent({ name: null, description: 'Something real.' })).toBe(false);+ expect(isIndexableAgent({})).toBe(false);+ expect(isIndexableAgent(null)).toBe(false);+ });+});++describe('isIndexableAvatar', () => {+ it('keeps a generated avatar: the description and tags still tell it apart', () => {+ expect(+ isIndexableAvatar({+ name: 'Shaw',+ description: 'Rigged, walk-ready avatar, forged on three.ws.',+ }),+ ).toBe(true);+ // A distinctive name is enough on its own.+ expect(isIndexableAvatar({ name: 'Roman Legionary', description: null })).toBe(true);+ });++ it('rejects a default name with nothing written about it', () => {+ expect(isIndexableAvatar({ name: 'My First Agent', description: null, alt_text: null })).toBe(+ false,+ );+ expect(isIndexableAvatar({ name: 'Avatar', description: '', alt_text: '' })).toBe(false);+ expect(isIndexableAvatar({ name: 'Untitled', description: null })).toBe(false);+ });++ it('accepts a default name once someone describes it', () => {+ expect(isIndexableAvatar({ name: 'Avatar', description: 'My studio mascot.' })).toBe(true);+ expect(isIndexableAvatar({ name: 'Avatar', alt_text: 'A chrome fox in a raincoat.' })).toBe(true);+ });+});diff --git a/tests/sitemap-type.test.js b/tests/sitemap-type.test.jsindex 0809e4984..421c88c59 100644--- a/tests/sitemap-type.test.js+++ b/tests/sitemap-type.test.js@@ -88,8 +88,14 @@ beforeEach(() => {db.queries = [];db.fail = null;db.rows = new Map([- ['agent_identities', [{ id: 'agent-1', updated_at: AT, created_at: AT }]],- ['avatars', [{ id: 'avatar-1', updated_at: null, created_at: AT }]],+ [+ 'agent_identities',+ [{ id: 'agent-1', name: 'Atlas', description: 'Watches markets.', updated_at: AT, created_at: AT }],+ ],+ [+ 'avatars',+ [{ id: 'avatar-1', name: 'Nova', description: 'A chrome scout.', alt_text: null, updated_at: null, created_at: AT }],+ ],['widgets', [{ id: 'widget-1', updated_at: AT, created_at: AT }]],['users u', [{ username: 'realhuman', updated_at: AT, created_at: AT }]],]);@@ -141,6 +147,32 @@ describe('GET /sitemap/<type>.xml: the six real types', () => {expect(res.body).toContain('<loc>https://three.ws/u/a&b<c>d"e'f</loc>');});+ it('leaves untouched onboarding rows out of the agents and avatars files', async () => {+ // Same name, same description, different uuid: nothing a crawler can use+ // to tell one from the next, so neither belongs in a sitemap.+ db.rows.set('agent_identities', [+ { id: 'real-1', name: 'Atlas', description: 'Watches markets.', updated_at: AT, created_at: AT },+ {+ id: 'starter-1',+ name: 'My First Agent',+ description:+ 'A friendly starter agent. Edit the personality and attach a 3D avatar \u2014 it goes live immediately.',+ updated_at: AT,+ created_at: AT,+ },+ ]);+ db.rows.set('avatars', [+ { id: 'real-2', name: 'Nova', description: 'A chrome scout.', alt_text: null, updated_at: AT, created_at: AT },+ { id: 'blank-2', name: 'Avatar', description: null, alt_text: null, updated_at: AT, created_at: AT },+ ]);++ const agents = locs((await call('agents')).body);+ expect(agents).toEqual(['https://three.ws/agents/real-1']);++ const avatars = locs((await call('avatars')).body);+ expect(avatars).toEqual(['https://three.ws/avatars/real-2']);+ });+it('drains the news archive newest month first', async () => {const found = locs((await call('news')).body);expect(found).toEqual([