Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(seo): make entity pages readable to a crawler and stop submitting the untouched ones

nirholas37b189ffef20
diff --git a/api/_lib/crawler-page.js b/api/_lib/crawler-page.js
new file mode 100644
index 000000000..050db9da3
--- /dev/null
+++ b/api/_lib/crawler-page.js
@@ -0,0 +1,274 @@
+/**
+ * Server-rendered entity pages for crawlers.
+ * -----------------------------------------
+ * /avatars/:id and /agents/:id are single-page-app routes: the HTML a browser
+ * receives is an empty shell and the entity is fetched client-side. vercel.json
+ * routes known bot User-Agents to an SSR handler instead so a shared link
+ * unfurls, and so a search engine has something to index.
+ *
+ * The first version of that SSR page carried only <meta> tags: the visible body
+ * was a spinner, the words lived inside <noscript>, and the last line was
+ * `location.replace(<this same URL>)`. That combination is invisible to a
+ * search engine in the worst possible way:
+ *
+ * - A rendering crawler (Googlebot, bingbot, Applebot) runs the script. The
+ * replace() targets the URL it is already on and the UA still matches the
+ * bot branch, so the response is byte-identical and the navigation repeats:
+ * a self-reload loop, and a URL the crawler may bucket as a redirect.
+ * - Rendering also drops <noscript>, so the only real text on the page is
+ * removed before indexing. Every entity URL then renders as the same
+ * spinner over the same "Loading…" line, i.e. one duplicate cluster tens of
+ * thousands of pages wide with no unique content to tell them apart.
+ *
+ * So the renderers here put the entity's real content in the body, and the
+ * self-redirect is emitted only for crawlers that never index (link unfurlers
+ * like Embedly or Iframely, where bouncing a human who arrives with a scraper
+ * UA is still the friendly outcome). Search engines get content and no script.
+ *
+ * Content parity is what keeps this out of cloaking territory: a crawler sees
+ * the same name, description, tags, owner, and imagery the interactive page
+ * shows, minus the WebGL viewport it cannot run.
+ */
+
+// Crawlers that index (and, for the big three, render JS before indexing).
+// These must never be handed a self-redirect or a content-free body.
+const SEARCH_CRAWLERS =
+ /(Googlebot|Google-InspectionTool|GoogleOther|Storebot-Google|Google-Extended|bingbot|BingPreview|Applebot|DuckDuckBot|DuckAssistBot|YandexBot|Baiduspider|PetalBot|SeznamBot|Amazonbot|Bytespider|CCBot|GPTBot|OAI-SearchBot|ChatGPT-User|ClaudeBot|Claude-SearchBot|Claude-User|PerplexityBot|Perplexity-User|MistralAI-User|cohere-ai|YouBot)/i;
+
+/**
+ * True when this User-Agent belongs to a crawler that builds a search or
+ * answer index, as opposed to a scraper that only wants unfurl metadata.
+ *
+ * @param {string|undefined|null} ua
+ * @returns {boolean}
+ */
+export function isSearchCrawler(ua) {
+ return SEARCH_CRAWLERS.test(String(ua || ''));
+}
+
+/** HTML-escape a value for use in text or a double-quoted attribute. */
+export function esc(s) {
+ return String(s ?? '').replace(
+ /[&<>"']/g,
+ (c) => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' })[c],
+ );
+}
+
+// One stylesheet for every crawler page. Inline because the page must be
+// self-contained: it is served from an /api route and a crawler that fetches
+// no subresources should still get a page that reads correctly.
+const STYLES = `
+ :root{color-scheme:dark}
+ *,*::before,*::after{box-sizing:border-box}
+ html,body{margin:0;padding:0;background:#06070a;color:#e7e9ee;
+ font:16px/1.6 Inter,system-ui,-apple-system,"Segoe UI",sans-serif;
+ -webkit-font-smoothing:antialiased}
+ a{color:#9fd0ff;text-decoration:none}
+ a:hover,a:focus-visible{text-decoration:underline;text-underline-offset:3px}
+ a:focus-visible{outline:2px solid #9fd0ff;outline-offset:3px;border-radius:4px}
+ main{max-width:56rem;margin:0 auto;padding:3rem 1.25rem 4rem}
+ nav[aria-label="Breadcrumb"]{font-size:.8125rem;color:#8b93a7;margin-bottom:1.5rem}
+ nav[aria-label="Breadcrumb"] ol{list-style:none;display:flex;flex-wrap:wrap;gap:.5rem;margin:0;padding:0}
+ nav[aria-label="Breadcrumb"] li+li::before{content:"/";margin-right:.5rem;color:#4a5164}
+ h1{font-size:clamp(1.75rem,4vw,2.5rem);line-height:1.15;margin:0 0 .75rem;letter-spacing:-.02em}
+ .lede{font-size:1.0625rem;color:#b6bccb;margin:0 0 2rem;max-width:44rem}
+ .card{display:block;width:100%;max-width:40rem;height:auto;aspect-ratio:1200/630;
+ border:1px solid #1d2230;border-radius:14px;background:#0b0d13;margin:0 0 2rem}
+ dl{display:grid;grid-template-columns:minmax(6rem,10rem) 1fr;gap:.5rem 1.5rem;margin:0 0 2rem;
+ font-size:.9375rem}
+ dt{color:#8b93a7}
+ dd{margin:0;color:#e7e9ee}
+ ul.tags{list-style:none;display:flex;flex-wrap:wrap;gap:.5rem;margin:0;padding:0}
+ ul.tags a{display:inline-block;padding:.15rem .6rem;border:1px solid #232838;border-radius:999px;
+ font-size:.8125rem;color:#b6bccb}
+ .actions{display:flex;flex-wrap:wrap;gap:.75rem;margin:0 0 2.5rem}
+ .actions a{display:inline-block;padding:.6rem 1.1rem;border-radius:10px;border:1px solid #232838;
+ background:#0e1119;color:#e7e9ee;font-size:.9375rem}
+ .actions a.primary{background:#e7e9ee;color:#06070a;border-color:#e7e9ee;font-weight:600}
+ footer{border-top:1px solid #171b26;padding-top:1.5rem;font-size:.875rem;color:#8b93a7}
+ footer ul{list-style:none;display:flex;flex-wrap:wrap;gap:1rem;margin:.5rem 0 0;padding:0}
+ @media(max-width:36rem){dl{grid-template-columns:1fr;gap:.15rem}dt{margin-top:.75rem}}
+`;
+
+/** Breadcrumb markup plus the matching schema.org ListItem entries. */
+function breadcrumb(trail, origin) {
+ const items = trail
+ .map((c, i) => {
+ const label = esc(c.name);
+ const inner = i === trail.length - 1 ? `<span aria-current="page">${label}</span>` : `<a href="${esc(c.path)}">${label}</a>`;
+ return `<li>${inner}</li>`;
+ })
+ .join('');
+ const ld = trail.map((c, i) => ({
+ '@type': 'ListItem',
+ position: i + 1,
+ name: c.name,
+ item: `${origin}${c.path}`,
+ }));
+ return { html: `<nav aria-label="Breadcrumb"><ol>${items}</ol></nav>`, ld };
+}
+
+/**
+ * Render a crawler-facing entity page: real content in the body, self-canonical
+ * in the head, structured data, and no self-redirect for an indexing crawler.
+ *
+ * @param {object} o
+ * @param {string} o.title entity name, unescaped
+ * @param {string} o.desc one-paragraph summary, unescaped
+ * @param {string} o.pageUrl absolute canonical URL
+ * @param {string} o.ogImage absolute URL of the 1200x630 card
+ * @param {string} o.origin site origin
+ * @param {string} o.ogType Open Graph object type
+ * @param {string} o.frameButton Farcaster frame button label
+ * @param {{name:string,path:string}[]} o.trail breadcrumb, root first
+ * @param {{term:string,detail:string}[]} [o.facts] definition rows, unescaped
+ * @param {{label:string,href:string}[]} [o.tags] tag chips
+ * @param {{label:string,href:string,primary?:boolean}[]} [o.actions] links out
+ * @param {{label:string,href:string}[]} [o.related] footer links
+ * @param {object} [o.jsonLd] entity node merged into the @graph
+ * @param {boolean} [o.redirect] emit the bounce-to-SPA script
+ * @param {boolean} [o.noindex] ask crawlers not to index this entity
+ * @returns {string} complete HTML document
+ */
+export function renderCrawlerPage({
+ title,
+ desc,
+ pageUrl,
+ ogImage,
+ origin,
+ ogType = 'website',
+ frameButton = 'Open on three.ws',
+ trail = [],
+ facts = [],
+ tags = [],
+ actions = [],
+ related = [],
+ jsonLd = null,
+ redirect = false,
+ noindex = false,
+}) {
+ const t = esc(title);
+ const d = esc(desc);
+ const crumbs = breadcrumb(trail, origin);
+ const path = new URL(pageUrl).pathname;
+
+ const graph = [];
+ if (jsonLd) graph.push(jsonLd);
+ graph.push({ '@type': 'BreadcrumbList', itemListElement: crumbs.ld });
+
+ const factsHtml = facts.length
+ ? `<dl>${facts.map((f) => `<dt>${esc(f.term)}</dt><dd>${f.detail}</dd>`).join('')}</dl>`
+ : '';
+ const tagsHtml = tags.length
+ ? `<ul class="tags">${tags.map((g) => `<li><a href="${esc(g.href)}">${esc(g.label)}</a></li>`).join('')}</ul>`
+ : '';
+ const actionsHtml = actions.length
+ ? `<div class="actions">${actions
+ .map((a) => `<a href="${esc(a.href)}"${a.primary ? ' class="primary"' : ''}>${esc(a.label)}</a>`)
+ .join('')}</div>`
+ : '';
+ const relatedHtml = related.length
+ ? `<ul>${related.map((r) => `<li><a href="${esc(r.href)}">${esc(r.label)}</a></li>`).join('')}</ul>`
+ : '';
+
+ return `<!doctype html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <title>${t} · three.ws</title>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover">
+ <meta name="description" content="${d}">
+ <meta name="theme-color" content="#06070a">
+ <meta name="robots" content="${noindex ? 'noindex, follow' : 'index, follow, max-image-preview:large, max-snippet:-1'}">
+
+ <meta property="og:type" content="${esc(ogType)}">
+ <meta property="og:site_name" content="three.ws">
+ <meta property="og:title" content="${t} · three.ws">
+ <meta property="og:description" content="${d}">
+ <meta property="og:url" content="${esc(pageUrl)}">
+ <meta property="og:image" content="${esc(ogImage)}">
+ <meta property="og:image:width" content="1200">
+ <meta property="og:image:height" content="630">
+ <meta property="og:image:alt" content="${t} on three.ws">
+
+ <meta name="twitter:card" content="summary_large_image">
+ <meta name="twitter:title" content="${t} · three.ws">
+ <meta name="twitter:description" content="${d}">
+ <meta name="twitter:image" content="${esc(ogImage)}">
+ <meta name="twitter:creator" content="@trythreews">
+
+ <meta property="fc:frame" content="vNext">
+ <meta property="fc:frame:image" content="${esc(ogImage)}">
+ <meta property="fc:frame:image:aspect_ratio" content="1.91:1">
+ <meta property="fc:frame:button:1" content="${esc(frameButton)}">
+ <meta property="fc:frame:button:1:action" content="link">
+ <meta property="fc:frame:button:1:target" content="${esc(pageUrl)}">
+
+ ${noindex ? '' : `<link rel="canonical" href="${esc(pageUrl)}">`}
+ <link rel="shortcut icon" href="/favicon.ico">
+ <style>${STYLES}</style>
+ <script type="application/ld+json">${JSON.stringify({ '@context': 'https://schema.org', '@graph': graph }).replace(/</g, '\\u003c')}</script>
+</head>
+<body>
+ <main>
+ ${crumbs.html}
+ <h1>${t}</h1>
+ <p class="lede">${d}</p>
+ <img class="card" src="${esc(ogImage)}" width="1200" height="630" alt="${t} on three.ws" loading="eager">
+ ${actionsHtml}
+ ${factsHtml}
+ ${tagsHtml}
+ <footer>
+ <p>${t} lives on three.ws, the agent layer for the open web: build a 3D avatar, give it an identity, and embed it anywhere.</p>
+ ${relatedHtml}
+ </footer>
+ </main>
+${redirect ? ` <script>(function(){window.location.replace(${JSON.stringify(path)});})()</script>\n` : ''}</body>
+</html>`;
+}
+
+/**
+ * Render the page a crawler gets for an id that names nothing public: gone,
+ * private, or never existed. Served with a 404 so the URL leaves the index
+ * instead of being logged as a redirect to an unrelated listing.
+ *
+ * @param {object} o
+ * @param {string} o.heading unescaped
+ * @param {string} o.message unescaped
+ * @param {string} o.origin
+ * @param {{label:string,href:string}[]} o.actions
+ * @param {string} [o.redirect] path to bounce a non-indexing scraper to
+ * @returns {string} complete HTML document
+ */
+export function renderCrawlerNotFound({ heading, message, origin, actions = [], redirect = '' }) {
+ const h = esc(heading);
+ const m = esc(message);
+ return `<!doctype html>
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <title>${h} · three.ws</title>
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <meta name="description" content="${m}">
+ <meta name="robots" content="noindex, follow">
+ <meta name="theme-color" content="#06070a">
+ <meta property="og:type" content="website">
+ <meta property="og:site_name" content="three.ws">
+ <meta property="og:title" content="${h} · three.ws">
+ <meta property="og:description" content="${m}">
+ <meta property="og:image" content="${esc(origin)}/og-image.png">
+ <meta name="twitter:card" content="summary_large_image">
+ <link rel="shortcut icon" href="/favicon.ico">
+ <style>${STYLES}</style>
+</head>
+<body>
+ <main>
+ <h1>${h}</h1>
+ <p class="lede">${m}</p>
+ <div class="actions">${actions
+ .map((a, i) => `<a href="${esc(a.href)}"${i === 0 ? ' class="primary"' : ''}>${esc(a.label)}</a>`)
+ .join('')}</div>
+ </main>
+${redirect ? ` <script>(function(){window.location.replace(${JSON.stringify(redirect)});})()</script>\n` : ''}</body>
+</html>`;
+}
diff --git a/api/_lib/indexable-entity.js b/api/_lib/indexable-entity.js
new file mode 100644
index 000000000..4fabdec13
--- /dev/null
+++ b/api/_lib/indexable-entity.js
@@ -0,0 +1,90 @@
+/**
+ * Which avatars and agents are worth asking a search engine to index.
+ * ------------------------------------------------------------------
+ * Onboarding creates an entity before its owner has said anything about it, and
+ * a lot of owners stop there. Measured 2026-09-07: 1,163 of the 3,397 public
+ * agents were still named "My First Agent" and 1,158 of those still carried the
+ * starter description verbatim. Those pages are not thin, they are *identical*,
+ * and no amount of server-side rendering can tell them apart.
+ *
+ * Submitting them costs twice. They cannot rank, and a duplicate cluster that
+ * large drags the pages that could rank down with it (Search Console filed
+ * 18,768 URLs under "Duplicate without user-selected canonical" before this).
+ *
+ * So the untouched ones stay out of the sitemap and their crawler page is served
+ * `noindex, follow`. Nothing else changes: the page still renders, still unfurls
+ * with the entity's own card, and still links onward. The moment an owner gives
+ * it a name or a description it becomes indexable, with no migration and no
+ * backfill. That is why the rule reads the row rather than storing a flag.
+ *
+ * Both the sitemap and the crawler pages import from here so the two can never
+ * disagree about what is indexable.
+ */
+
+/** Fold case, collapse whitespace, and normalise dash glyphs for comparison. */
+function normalize(s) {
+ return String(s ?? '')
+ .replace(/[\u2010-\u2015]/g, '-')
+ .trim()
+ .replace(/\s+/g, ' ')
+ .toLowerCase();
+}
+
+// Names the product hands out when nobody has chosen one. An entity keeping its
+// default name is evidence of an untouched row, not of a name someone picked.
+const DEFAULT_AGENT_NAMES = new Set([
+ 'my first agent',
+ 'my agent',
+ 'agent',
+ 'new agent',
+ 'untitled',
+ 'untitled agent',
+]);
+
+const DEFAULT_AVATAR_NAMES = new Set([
+ 'my first agent',
+ 'my avatar',
+ 'avatar',
+ 'new avatar',
+ 'untitled',
+ 'untitled avatar',
+]);
+
+// The description onboarding writes into a brand-new agent. Matched on its
+// opening words so a trailing edit, or the dash glyph in the stored copy, does
+// not let an untouched row through.
+const STARTER_AGENT_DESCRIPTION = 'a friendly starter agent';
+
+/**
+ * True when this agent has enough of its own identity to deserve its own entry
+ * in a search index.
+ *
+ * @param {{name?: string|null, description?: string|null}} agent
+ * @returns {boolean}
+ */
+export function isIndexableAgent(agent) {
+ const name = normalize(agent?.name);
+ if (!name) return false;
+ const description = normalize(agent?.description);
+ const untouchedName = DEFAULT_AGENT_NAMES.has(name);
+ const untouchedDescription = !description || description.startsWith(STARTER_AGENT_DESCRIPTION);
+ // Either half being the owner's own work is enough: a custom description
+ // under a default name still describes something specific, and vice versa.
+ return !(untouchedName && untouchedDescription);
+}
+
+/**
+ * True when this avatar has enough of its own identity to deserve its own entry
+ * in a search index. Avatars carry a generated description and a tag set even
+ * when the owner writes nothing, so only a default name with no prose at all
+ * counts as untouched.
+ *
+ * @param {{name?: string|null, description?: string|null, alt_text?: string|null}} avatar
+ * @returns {boolean}
+ */
+export function isIndexableAvatar(avatar) {
+ const name = normalize(avatar?.name);
+ if (!name) return false;
+ if (!DEFAULT_AVATAR_NAMES.has(name)) return true;
+ return Boolean(normalize(avatar?.description) || normalize(avatar?.alt_text));
+}
diff --git a/api/agent-detail-og.js b/api/agent-detail-og.js
index cad3daa9b..2cbda69fc 100644
--- a/api/agent-detail-og.js
+++ b/api/agent-detail-og.js
@@ -1,148 +1,229 @@
/**
- * SSR OG page for agent detail (/agents/:id)
+ * Crawler page for agent detail (/agents/:id)
* -------------------------------------------
* GET /api/agent-detail-og?id=<agentId>
*
- * Wired via vercel.json: when a social crawler hits /agents/<uuid>,
- * a User-Agent "has" condition rewrites to this endpoint. Returns a minimal
- * HTML page with OG + Twitter Card + Farcaster Frame meta so shared agent
- * links unfurl with a real image and description.
+ * Wired via vercel.json: when a crawler User-Agent hits /agents/<uuid>, a
+ * "has" condition rewrites to this endpoint. Real browsers never reach it, so
+ * this is the only version of the page a search engine or a link unfurler ever
+ * sees, and it has to carry the agent's real content rather than meta tags over
+ * a spinner. See the header of api/_lib/crawler-page.js.
*
- * Real browsers never reach this route — the rewrite only fires for known
- * bot User-Agents. Mirrors the pattern in api/app-og.js.
+ * Mirrors api/avatar-detail-og.js.
*/
import { sql } from './_lib/db.js';
import { cors, method, wrap } from './_lib/http.js';
import { env } from './_lib/env.js';
import { isUuid } from './_lib/validate.js';
+import { esc, isSearchCrawler, renderCrawlerPage, renderCrawlerNotFound } from './_lib/crawler-page.js';
+import { isIndexableAgent } from './_lib/indexable-entity.js';
export default wrap(async (req, res) => {
if (cors(req, res, { methods: 'GET,OPTIONS' })) return;
- // Crawler-only SSR read. Without this, a POST answered with the 302
- // passthrough instead of the 405 the advertised Allow set promises.
+ // Crawler-only SSR read. Without this, a POST answered with a page instead
+ // of the 405 the advertised Allow set promises.
if (!method(req, res, ['GET'])) return;
const url = new URL(req.url, 'http://x');
const agentId = url.searchParams.get('id');
const origin = env.APP_ORIGIN || 'https://three.ws';
+ const ua = req.headers?.['user-agent'];
+ // Only a non-indexing scraper gets bounced to the app. An indexing crawler
+ // runs that script, lands back on this same URL behind the same UA branch,
+ // and loops on an identical response.
+ const redirect = !isSearchCrawler(ua);
if (!agentId || !isUuid(agentId)) {
- return passthrough(res, origin);
+ return generic(res, origin, agentId, redirect);
}
let agent;
try {
[agent] = await sql`
- SELECT i.id, i.name, i.description, i.skills
+ SELECT i.id, i.name, i.description, i.skills, i.home_url,
+ i.erc8004_agent_id, i.chain_id, i.created_at, i.updated_at
FROM agent_identities i
- WHERE i.id = ${agentId} AND i.deleted_at IS NULL
+ WHERE i.id = ${agentId} AND i.deleted_at IS NULL AND i.is_public = true
LIMIT 1
`;
} catch {
- return passthrough(res, origin);
+ // A read failure is ours, not the URL's. 503 asks the crawler to come
+ // back; a 404 or a redirect here would drop a live page from the index.
+ return unavailable(res, origin);
}
- if (!agent) return passthrough(res, origin);
+ if (!agent) {
+ // Agents and avatars have separate uuid spaces and links cross over
+ // (see resolveEntity in src/avatar-page.js). Send the crawler to the
+ // path that actually holds the entity instead of reporting nothing.
+ let avatar = null;
+ try {
+ [avatar] = await sql`
+ SELECT id FROM avatars
+ WHERE id = ${agentId} AND deleted_at IS NULL AND visibility = 'public' LIMIT 1
+ `;
+ } catch {
+ return unavailable(res, origin);
+ }
+ if (avatar) {
+ res.statusCode = 301;
+ res.setHeader('location', `${origin}/avatars/${agentId}`);
+ res.setHeader('cache-control', 'public, max-age=300');
+ res.end();
+ return;
+ }
+ return gone(res, origin);
+ }
const title = agent.name || 'Agent';
- const baseDesc = agent.description || 'An AI agent on three.ws — with a body, a place, and an identity.';
- const skills = agent.skills || [];
+ const baseDesc =
+ agent.description || 'An AI agent on three.ws, with a body, a place, and an identity.';
+ const skills = Array.isArray(agent.skills) ? agent.skills : [];
const skillSuffix = skills.length
? ` Skills: ${skills.slice(0, 4).join(', ')}${skills.length > 4 ? '…' : ''}.`
: '';
const desc = baseDesc + skillSuffix;
- // Unfurl with the dynamic wallet TRADING CARD (api/og/agent.js) — the same
+ // Unfurl with the dynamic wallet TRADING CARD (api/og/agent.js) - the same
// living card shown on the page (avatar, vanity address, live net worth, P&L,
// reputation tier, finish), rendered from real data. The card embeds the
// avatar itself, so a shared link previews with identity + wallet, not a bare
// thumbnail. The card endpoint falls back to the brand image if it can't render.
const ogImage = `${origin}/api/og/agent?id=${encodeURIComponent(agentId)}`;
-
const pageUrl = `${origin}/agents/${agentId}`;
+ const facts = [];
+ if (skills.length) {
+ facts.push({
+ term: 'Skills',
+ detail: skills.slice(0, 12).map((s) => esc(s)).join(', '),
+ });
+ }
+ if (agent.erc8004_agent_id) {
+ facts.push({
+ term: 'On-chain identity',
+ detail: `ERC-8004 agent #${esc(agent.erc8004_agent_id)}${agent.chain_id ? ` on chain ${esc(agent.chain_id)}` : ''}`,
+ });
+ }
+ if (agent.home_url && /^https?:\/\//i.test(agent.home_url)) {
+ const h = esc(agent.home_url);
+ facts.push({ term: 'Home', detail: `<a href="${h}" rel="nofollow noopener">${h}</a>` });
+ }
+ if (agent.created_at) facts.push({ term: 'Registered', detail: isoDay(agent.created_at) });
+
res.statusCode = 200;
res.setHeader('content-type', 'text/html; charset=utf-8');
res.setHeader('cache-control', 'public, max-age=60, s-maxage=600, stale-while-revalidate=3600');
- res.end(renderHtml({ agentId, title, desc, pageUrl, ogImage, origin }));
+ res.end(
+ renderCrawlerPage({
+ title,
+ desc,
+ pageUrl,
+ ogImage,
+ origin,
+ ogType: 'profile',
+ frameButton: `Meet ${title}`,
+ redirect,
+ // An untouched onboarding row is identical to every other one, so it
+ // unfurls and renders but never asks to be indexed.
+ noindex: !isIndexableAgent(agent),
+ trail: [
+ { name: 'Home', path: '/' },
+ { name: 'Agents', path: '/agents' },
+ { name: title, path: `/agents/${agentId}` },
+ ],
+ facts,
+ // No skill chips: the directory filters on ?q= over name and
+ // description, not on a skill facet, so a chip would look like a
+ // filter and return an unfiltered list. The skills are in the facts
+ // row and in featureList below.
+ actions: [
+ { label: 'View in AR', href: `/agents/${agentId}/ar`, primary: true },
+ { label: 'Agent directory', href: '/agents' },
+ { label: 'Build an agent', href: '/create' },
+ ],
+ related: [
+ { label: 'Agent directory', href: '/agents' },
+ { label: 'Agent economy', href: '/agent-economy' },
+ { label: 'Agent marketplace', href: '/marketplace' },
+ { label: 'Avatar gallery', href: '/gallery' },
+ ],
+ jsonLd: {
+ '@type': 'SoftwareApplication',
+ '@id': pageUrl,
+ name: title,
+ description: desc,
+ url: pageUrl,
+ image: ogImage,
+ applicationCategory: 'AI agent',
+ operatingSystem: 'Web',
+ ...(skills.length ? { featureList: skills.slice(0, 12) } : {}),
+ ...(agent.created_at ? { dateCreated: isoDay(agent.created_at) } : {}),
+ ...(agent.updated_at ? { dateModified: isoDay(agent.updated_at) } : {}),
+ publisher: { '@type': 'Organization', name: 'three.ws', url: origin },
+ },
+ }),
+ );
});
-function passthrough(res, origin) {
- res.statusCode = 302;
- res.setHeader('location', `${origin}/agents`);
- res.setHeader('cache-control', 'no-cache');
- res.end();
+/** An id we cannot read: keep the unfurl useful, keep the URL out of the index. */
+function generic(res, origin, agentId, redirect) {
+ const target = agentId ? `/agents/${encodeURIComponent(agentId)}` : '/agents';
+ res.statusCode = 200;
+ res.setHeader('content-type', 'text/html; charset=utf-8');
+ res.setHeader('cache-control', 'public, max-age=60, s-maxage=600');
+ res.end(
+ renderCrawlerNotFound({
+ heading: 'Agent on three.ws',
+ message: 'Open this agent on three.ws to see its body, its wallet, and what it can do.',
+ origin,
+ actions: [
+ { label: 'Open the agent', href: target },
+ { label: 'Agent directory', href: '/agents' },
+ ],
+ redirect: redirect ? target : '',
+ }),
+ );
}
-function renderHtml({ agentId, title, desc, pageUrl, ogImage, origin }) {
- const t = esc(title);
- const d = esc(desc);
- const agentUrl = `/agents/${agentId}`;
- return `<!doctype html>
-<html lang="en">
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <title>${t} — three.ws</title>
- <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover">
- <meta name="description" content="${d}">
- <meta name="theme-color" content="#06070a">
-
- <meta property="og:type" content="profile">
- <meta property="og:site_name" content="three.ws">
- <meta property="og:title" content="${t} — three.ws">
- <meta property="og:description" content="${d}">
- <meta property="og:url" content="${esc(pageUrl)}">
- <meta property="og:image" content="${esc(ogImage)}">
- <meta property="og:image:width" content="1200">
- <meta property="og:image:height" content="630">
- <meta property="og:image:alt" content="${t} on three.ws">
-
- <meta name="twitter:card" content="summary_large_image">
- <meta name="twitter:title" content="${t} — three.ws">
- <meta name="twitter:description" content="${d}">
- <meta name="twitter:image" content="${esc(ogImage)}">
- <meta name="twitter:creator" content="@trythreews">
-
- <meta property="fc:frame" content="vNext">
- <meta property="fc:frame:image" content="${esc(ogImage)}">
- <meta property="fc:frame:image:aspect_ratio" content="1.91:1">
- <meta property="fc:frame:button:1" content="Meet ${t}">
- <meta property="fc:frame:button:1:action" content="link">
- <meta property="fc:frame:button:1:target" content="${esc(pageUrl)}">
-
- <link rel="canonical" href="${esc(pageUrl)}">
- <link rel="shortcut icon" href="/favicon.ico">
-
- <style>
- html,body{margin:0;padding:0;background:#06070a;color:#e0e0e0;font-family:Inter,system-ui,sans-serif;height:100%}
- .shell{display:grid;place-items:center;min-height:100vh;text-align:center;padding:2rem;gap:1rem}
- .shell a{color:#e0e0e0;text-decoration:underline;text-underline-offset:3px}
- .spinner{width:28px;height:28px;border:2px solid rgba(255,255,255,0.1);border-top-color:rgba(255,255,255,0.6);border-radius:50%;animation:spin 0.9s linear infinite;margin:0 auto}
- @keyframes spin{to{transform:rotate(360deg)}}
- p{margin:0;color:rgba(255,255,255,0.5);font-size:14px}
- </style>
-</head>
-<body>
- <noscript>
- <div class="shell">
- <h1>${t}</h1>
- <p>${d}</p>
- <p><a href="${esc(pageUrl)}">View agent</a> · <a href="${esc(origin)}/agents">Browse agents</a></p>
- </div>
- </noscript>
- <div class="shell" aria-live="polite">
- <div class="spinner" aria-hidden="true"></div>
- <p>Loading ${t}…</p>
- </div>
- <script>(function(){window.location.replace(${JSON.stringify(agentUrl)});})()</script>
-</body>
-</html>`;
+/** Nothing public lives at this id. 404 so the URL leaves the index cleanly. */
+function gone(res, origin) {
+ res.statusCode = 404;
+ res.setHeader('content-type', 'text/html; charset=utf-8');
+ res.setHeader('cache-control', 'public, max-age=300');
+ res.end(
+ renderCrawlerNotFound({
+ heading: 'This agent is not public',
+ message:
+ 'The agent behind this link was retired, or its owner keeps it private. The directory lists the ones that are live.',
+ origin,
+ actions: [
+ { label: 'Agent directory', href: '/agents' },
+ { label: 'Build an agent', href: '/create' },
+ ],
+ }),
+ );
}
-function esc(s) {
- return String(s ?? '').replace(/[&<>"']/g, (c) =>
- ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' })[c],
+/** Our read failed. Ask the crawler to retry rather than losing the page. */
+function unavailable(res, origin) {
+ res.statusCode = 503;
+ res.setHeader('content-type', 'text/html; charset=utf-8');
+ res.setHeader('retry-after', '120');
+ res.setHeader('cache-control', 'no-store');
+ res.end(
+ renderCrawlerNotFound({
+ heading: 'Agent temporarily unavailable',
+ message: 'three.ws could not load this agent just now. Please try again in a moment.',
+ origin,
+ actions: [{ label: 'Agent directory', href: '/agents' }],
+ }),
);
}
+
+/** YYYY-MM-DD, the only precision a crawler page needs. */
+function isoDay(v) {
+ const t = v instanceof Date ? v : new Date(v);
+ return Number.isNaN(t.getTime()) ? '' : t.toISOString().slice(0, 10);
+}
diff --git a/api/avatar-detail-og.js b/api/avatar-detail-og.js
index ad38609b9..47e02099a 100644
--- a/api/avatar-detail-og.js
+++ b/api/avatar-detail-og.js
@@ -1,40 +1,53 @@
/**
- * SSR OG page for avatar detail (/avatars/:id)
- * --------------------------------------------
+ * Crawler page for avatar detail (/avatars/:id)
+ * ---------------------------------------------
* GET /api/avatar-detail-og?id=<avatarId>
*
- * Wired via vercel.json: when a social crawler hits /avatars/<uuid>,
- * a User-Agent "has" condition rewrites to this endpoint. Returns a minimal
- * HTML page with OG + Twitter Card + Farcaster Frame meta so shared avatar
- * links unfurl with the avatar's real name, description, and rendered card.
+ * Wired via vercel.json: when a crawler User-Agent hits /avatars/<uuid>, a
+ * "has" condition rewrites to this endpoint. Real browsers never reach it, so
+ * this is the only version of the page a search engine or a link unfurler ever
+ * sees. It therefore has to carry the avatar's real content, not just meta
+ * tags: see the header of api/_lib/crawler-page.js for why a spinner plus a
+ * <noscript> block collapsed 45k avatar URLs into a single duplicate cluster.
*
- * Real browsers never reach this route - the rewrite only fires for known
- * bot User-Agents. Mirrors api/agent-detail-og.js.
+ * Mirrors api/agent-detail-og.js.
*/
import { sql } from './_lib/db.js';
import { cors, method, wrap } from './_lib/http.js';
import { env } from './_lib/env.js';
import { isUuid } from './_lib/validate.js';
+import { esc, isSearchCrawler, renderCrawlerPage, renderCrawlerNotFound } from './_lib/crawler-page.js';
+import { isIndexableAvatar } from './_lib/indexable-entity.js';
export default wrap(async (req, res) => {
if (cors(req, res, { methods: 'GET,OPTIONS' })) return;
- // Crawler-only SSR read. Without this, a POST answered with the 302
- // passthrough instead of the 405 the advertised Allow set promises.
+ // Crawler-only SSR read. Without this, a POST answered with a page instead
+ // of the 405 the advertised Allow set promises.
if (!method(req, res, ['GET'])) return;
const url = new URL(req.url, 'http://x');
const avatarId = url.searchParams.get('id');
const origin = env.APP_ORIGIN || 'https://three.ws';
-
+ const ua = req.headers?.['user-agent'];
+ // Only a non-indexing scraper gets bounced to the app. An indexing crawler
+ // runs that script, lands back on this same URL behind the same UA branch,
+ // and loops on an identical response.
+ const redirect = !isSearchCrawler(ua);
+
+ // Demo and legacy ids are not uuids. They still resolve in the app, so keep
+ // the bounce and a valid unfurl, but keep them out of the index rather than
+ // inventing a page for an entity we cannot read.
if (!avatarId || !isUuid(avatarId)) {
- return passthrough(res, origin);
+ return generic(res, origin, avatarId, redirect);
}
let avatar;
try {
[avatar] = await sql`
- SELECT a.id, a.name, a.description, a.tags, a.model_category,
+ SELECT a.id, a.name, a.description, a.alt_text, a.tags, a.model_category,
+ a.view_count, a.fork_count, a.parent_avatar_id,
+ a.created_at, a.updated_at,
u.username AS owner_username
FROM avatars a
LEFT JOIN users u ON u.id = a.owner_id AND u.deleted_at IS NULL
@@ -42,16 +55,43 @@ export default wrap(async (req, res) => {
LIMIT 1
`;
} catch {
- return passthrough(res, origin);
+ // A read failure is ours, not the URL's. 503 asks the crawler to come
+ // back; a 404 or a redirect here would drop a live page from the index.
+ return unavailable(res, origin);
}
- if (!avatar) return passthrough(res, origin);
+ if (!avatar) {
+ // Agents and avatars have separate uuid spaces and links cross over
+ // (see resolveEntity in src/avatar-page.js). Send the crawler to the
+ // path that actually holds the entity instead of reporting nothing.
+ let agent = null;
+ try {
+ [agent] = await sql`
+ SELECT id FROM agent_identities
+ WHERE id = ${avatarId} AND deleted_at IS NULL LIMIT 1
+ `;
+ } catch {
+ return unavailable(res, origin);
+ }
+ if (agent) {
+ res.statusCode = 301;
+ res.setHeader('location', `${origin}/agents/${avatarId}`);
+ res.setHeader('cache-control', 'public, max-age=300');
+ res.end();
+ return;
+ }
+ return gone(res, origin);
+ }
const title = avatar.name || 'Avatar';
const baseDesc =
- avatar.description || 'A 3D avatar on three.ws - rigged, animated, and ready to become an agent.';
- const tags = avatar.tags || [];
- const tagSuffix = tags.length ? ` Tags: ${tags.slice(0, 4).join(', ')}${tags.length > 4 ? '…' : ''}.` : '';
+ avatar.description ||
+ avatar.alt_text ||
+ 'A 3D avatar on three.ws - rigged, animated, and ready to become an agent.';
+ const tags = Array.isArray(avatar.tags) ? avatar.tags : [];
+ const tagSuffix = tags.length
+ ? ` Tags: ${tags.slice(0, 4).join(', ')}${tags.length > 4 ? '…' : ''}.`
+ : '';
const byline = avatar.owner_username ? ` By @${avatar.owner_username}.` : '';
const desc = baseDesc + byline + tagSuffix;
@@ -59,90 +99,148 @@ export default wrap(async (req, res) => {
// turntable render with name plate, from real data. The card endpoint falls
// back to the brand image if it can't render.
const ogImage = `${origin}/api/avatars/${encodeURIComponent(avatarId)}/og`;
-
const pageUrl = `${origin}/avatars/${avatarId}`;
+ const facts = [];
+ if (avatar.owner_username) {
+ const u = esc(avatar.owner_username);
+ facts.push({ term: 'Creator', detail: `<a href="/u/${u}" rel="author">@${u}</a>` });
+ }
+ if (avatar.model_category) facts.push({ term: 'Category', detail: esc(avatar.model_category) });
+ if (avatar.parent_avatar_id) {
+ const parent = esc(avatar.parent_avatar_id);
+ facts.push({ term: 'Remixed from', detail: `<a href="/avatars/${parent}">the original avatar</a>` });
+ }
+ if (Number(avatar.fork_count) > 0) {
+ facts.push({ term: 'Remixes', detail: `${Number(avatar.fork_count)} on three.ws` });
+ }
+ if (Number(avatar.view_count) > 0) {
+ facts.push({ term: 'Views', detail: String(Number(avatar.view_count)) });
+ }
+ if (avatar.created_at) facts.push({ term: 'Published', detail: isoDay(avatar.created_at) });
+
res.statusCode = 200;
res.setHeader('content-type', 'text/html; charset=utf-8');
res.setHeader('cache-control', 'public, max-age=60, s-maxage=600, stale-while-revalidate=3600');
- res.end(renderHtml({ avatarId, title, desc, pageUrl, ogImage, origin }));
+ res.end(
+ renderCrawlerPage({
+ title,
+ desc,
+ pageUrl,
+ ogImage,
+ origin,
+ ogType: 'profile',
+ frameButton: `Meet ${title}`,
+ redirect,
+ // An untouched onboarding row is identical to every other one, so it
+ // unfurls and renders but never asks to be indexed.
+ noindex: !isIndexableAvatar(avatar),
+ trail: [
+ { name: 'Home', path: '/' },
+ { name: 'Gallery', path: '/gallery' },
+ { name: title, path: `/avatars/${avatarId}` },
+ ],
+ facts,
+ tags: tags.slice(0, 12).map((tag) => ({
+ label: tag,
+ href: `/gallery?tag=${encodeURIComponent(tag)}`,
+ })),
+ actions: [
+ { label: 'View in AR', href: `/avatars/${avatarId}/ar`, primary: true },
+ { label: 'Browse the gallery', href: '/gallery' },
+ { label: 'Create your own', href: '/create' },
+ ],
+ related: [
+ { label: 'Avatar gallery', href: '/gallery' },
+ { label: 'Animation library', href: '/animations' },
+ { label: 'Build an embeddable widget', href: '/studio' },
+ { label: 'Agent directory', href: '/agents' },
+ ],
+ jsonLd: {
+ '@type': '3DModel',
+ '@id': pageUrl,
+ name: title,
+ description: desc,
+ url: pageUrl,
+ image: ogImage,
+ encodingFormat: 'model/gltf-binary',
+ isFamilyFriendly: true,
+ ...(tags.length ? { keywords: tags.join(', ') } : {}),
+ ...(avatar.created_at ? { dateCreated: isoDay(avatar.created_at) } : {}),
+ ...(avatar.updated_at ? { dateModified: isoDay(avatar.updated_at) } : {}),
+ ...(avatar.owner_username
+ ? {
+ creator: {
+ '@type': 'Person',
+ name: `@${avatar.owner_username}`,
+ url: `${origin}/u/${avatar.owner_username}`,
+ },
+ }
+ : {}),
+ publisher: { '@type': 'Organization', name: 'three.ws', url: origin },
+ },
+ }),
+ );
});
-function passthrough(res, origin) {
- res.statusCode = 302;
- res.setHeader('location', `${origin}/gallery`);
- res.setHeader('cache-control', 'no-cache');
- res.end();
+/** An id we cannot read: keep the unfurl useful, keep the URL out of the index. */
+function generic(res, origin, avatarId, redirect) {
+ const target = avatarId ? `/avatars/${encodeURIComponent(avatarId)}` : '/gallery';
+ res.statusCode = 200;
+ res.setHeader('content-type', 'text/html; charset=utf-8');
+ res.setHeader('cache-control', 'public, max-age=60, s-maxage=600');
+ res.end(
+ renderCrawlerNotFound({
+ heading: 'Avatar on three.ws',
+ message:
+ 'Open this avatar in the three.ws studio to view it in 3D, animate it, and embed it anywhere.',
+ origin,
+ actions: [
+ { label: 'Open the studio', href: target },
+ { label: 'Browse the gallery', href: '/gallery' },
+ ],
+ redirect: redirect ? target : '',
+ }),
+ );
}
-function renderHtml({ avatarId, title, desc, pageUrl, ogImage, origin }) {
- const t = esc(title);
- const d = esc(desc);
- const avatarUrl = `/avatars/${avatarId}`;
- return `<!doctype html>
-<html lang="en">
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <title>${t} · three.ws</title>
- <meta name="viewport" content="width=device-width, initial-scale=1.0, viewport-fit=cover">
- <meta name="description" content="${d}">
- <meta name="theme-color" content="#06070a">
-
- <meta property="og:type" content="profile">
- <meta property="og:site_name" content="three.ws">
- <meta property="og:title" content="${t} · three.ws">
- <meta property="og:description" content="${d}">
- <meta property="og:url" content="${esc(pageUrl)}">
- <meta property="og:image" content="${esc(ogImage)}">
- <meta property="og:image:width" content="1200">
- <meta property="og:image:height" content="630">
- <meta property="og:image:alt" content="${t} on three.ws">
-
- <meta name="twitter:card" content="summary_large_image">
- <meta name="twitter:title" content="${t} · three.ws">
- <meta name="twitter:description" content="${d}">
- <meta name="twitter:image" content="${esc(ogImage)}">
- <meta name="twitter:creator" content="@trythreews">
-
- <meta property="fc:frame" content="vNext">
- <meta property="fc:frame:image" content="${esc(ogImage)}">
- <meta property="fc:frame:image:aspect_ratio" content="1.91:1">
- <meta property="fc:frame:button:1" content="Meet ${t}">
- <meta property="fc:frame:button:1:action" content="link">
- <meta property="fc:frame:button:1:target" content="${esc(pageUrl)}">
-
- <link rel="canonical" href="${esc(pageUrl)}">
- <link rel="shortcut icon" href="/favicon.ico">
-
- <style>
- html,body{margin:0;padding:0;background:#06070a;color:#e0e0e0;font-family:Inter,system-ui,sans-serif;height:100%}
- .shell{display:grid;place-items:center;min-height:100vh;text-align:center;padding:2rem;gap:1rem}
- .shell a{color:#e0e0e0;text-decoration:underline;text-underline-offset:3px}
- .spinner{width:28px;height:28px;border:2px solid rgba(255,255,255,0.1);border-top-color:rgba(255,255,255,0.6);border-radius:50%;animation:spin 0.9s linear infinite;margin:0 auto}
- @keyframes spin{to{transform:rotate(360deg)}}
- p{margin:0;color:rgba(255,255,255,0.5);font-size:14px}
- </style>
-</head>
-<body>
- <noscript>
- <div class="shell">
- <h1>${t}</h1>
- <p>${d}</p>
- <p><a href="${esc(pageUrl)}">View avatar</a> · <a href="${esc(origin)}/gallery">Browse the gallery</a></p>
- </div>
- </noscript>
- <div class="shell" aria-live="polite">
- <div class="spinner" aria-hidden="true"></div>
- <p>Loading ${t}…</p>
- </div>
- <script>(function(){window.location.replace(${JSON.stringify(avatarUrl)});})()</script>
-</body>
-</html>`;
+/** Nothing public lives at this id. 404 so the URL leaves the index cleanly. */
+function gone(res, origin) {
+ res.statusCode = 404;
+ res.setHeader('content-type', 'text/html; charset=utf-8');
+ res.setHeader('cache-control', 'public, max-age=300');
+ res.end(
+ renderCrawlerNotFound({
+ heading: 'This avatar is not public',
+ message:
+ 'The avatar behind this link was removed, or its owner keeps it private. The gallery has thousands of others.',
+ origin,
+ actions: [
+ { label: 'Browse the gallery', href: '/gallery' },
+ { label: 'Create an avatar', href: '/create' },
+ ],
+ }),
+ );
}
-function esc(s) {
- return String(s ?? '').replace(/[&<>"']/g, (c) =>
- ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;', "'": '&#39;' })[c],
+/** Our read failed. Ask the crawler to retry rather than losing the page. */
+function unavailable(res, origin) {
+ res.statusCode = 503;
+ res.setHeader('content-type', 'text/html; charset=utf-8');
+ res.setHeader('retry-after', '120');
+ res.setHeader('cache-control', 'no-store');
+ res.end(
+ renderCrawlerNotFound({
+ heading: 'Avatar temporarily unavailable',
+ message: 'three.ws could not load this avatar just now. Please try again in a moment.',
+ origin,
+ actions: [{ label: 'Browse the gallery', href: '/gallery' }],
+ }),
);
}
+
+/** YYYY-MM-DD, the only precision a crawler page needs. */
+function isoDay(v) {
+ const t = v instanceof Date ? v : new Date(v);
+ return Number.isNaN(t.getTime()) ? '' : t.toISOString().slice(0, 10);
+}
diff --git a/api/sitemap/[type].js b/api/sitemap/[type].js
index 807838738..d7de42de0 100644
--- a/api/sitemap/[type].js
+++ b/api/sitemap/[type].js
@@ -17,6 +17,7 @@ import { reportServerError, redactUrl } from '../_lib/http.js';
import { getMonths, loadMonth } from '../_lib/news-archive-store.js';
import { storyPath } from '../../src/shared/news-links.js';
import { isSuppressed } from '../_lib/news-rights.js';
+import { isIndexableAgent, isIndexableAvatar } from '../_lib/indexable-entity.js';
const ORIGIN = env.APP_ORIGIN;
const MAX_URLS = 45_000;
@@ -155,35 +156,54 @@ async function coreSitemap() {
}
async function agentsSitemap() {
+ // Drop the untouched onboarding rows: 1,359 of the 3,397 public agents still
+ // carried the starter name and description on 2026-09-07, and they are
+ // duplicates of each other by construction (api/_lib/indexable-entity.js).
+ // The filter runs in JS rather than SQL so one definition of "indexable"
+ // serves both this file and the crawler page, which must agree. The
+ // predicate only tests whether the prose is empty and whether it opens with
+ // the starter copy, so a prefix is all it needs and the row stays small.
const rows = await sql`
- select id, updated_at, created_at
+ select id, name, left(description, 64) as description, updated_at, created_at
from agent_identities
where deleted_at is null and is_public = true
order by coalesce(updated_at, created_at) desc
limit ${MAX_URLS}
`;
- return rows.map((r) => ({
- loc: `${ORIGIN}/agents/${r.id}`,
- lastmod: fmtDate(r.updated_at || r.created_at),
- changefreq: 'weekly',
- priority: '0.7',
- }));
+ return rows
+ .filter(isIndexableAgent)
+ .map((r) => ({
+ loc: `${ORIGIN}/agents/${r.id}`,
+ lastmod: fmtDate(r.updated_at || r.created_at),
+ changefreq: 'weekly',
+ priority: '0.7',
+ }));
}
async function avatarsSitemap() {
+ // Same filter as agentsSitemap: an avatar still carrying its default name with
+ // nothing written about it cannot be told apart from the next one. It removes
+ // far less here (170 of 67,164 on 2026-09-07) because a generated avatar
+ // still gets its own description and tags. Filtering after the cap rather
+ // than widening the fetch is deliberate: the public set is already half again
+ // larger than the 45k ceiling, so the file is truncated either way and
+ // pulling 90k rows into memory to reclaim 0.25% of the slots is not a trade.
const rows = await sql`
- select id, updated_at, created_at
+ select id, name, left(description, 64) as description, left(alt_text, 64) as alt_text,
+ updated_at, created_at
from avatars
where deleted_at is null and visibility = 'public'
order by coalesce(updated_at, created_at) desc
limit ${MAX_URLS}
`;
- return rows.map((r) => ({
- loc: `${ORIGIN}/avatars/${r.id}`,
- lastmod: fmtDate(r.updated_at || r.created_at),
- changefreq: 'weekly',
- priority: '0.6',
- }));
+ return rows
+ .filter(isIndexableAvatar)
+ .map((r) => ({
+ loc: `${ORIGIN}/avatars/${r.id}`,
+ lastmod: fmtDate(r.updated_at || r.created_at),
+ changefreq: 'weekly',
+ priority: '0.6',
+ }));
}
async function widgetsSitemap() {
diff --git a/docs/ops/README.md b/docs/ops/README.md
index ec497f1c0..25f8eca6d 100644
--- a/docs/ops/README.md
+++ b/docs/ops/README.md
@@ -30,6 +30,7 @@ Start with [gcp-production.md](gcp-production.md); it is the complete operationa
| [runtime-flags.md](runtime-flags.md) | DB-backed feature flags that flip platform behavior without a redeploy. |
| [x402-discovery-listings.md](x402-discovery-listings.md) | Getting paid endpoints listed and ranked on x402scan, the Bazaar, and other directories. |
| [seo-keyword-plan.md](seo-keyword-plan.md) | Verified keyword landscape and content calendar (snapshot dated 2026-07-17). |
+| [search-indexing.md](search-indexing.md) | How pages reach a search index: the three serving paths, the crawler-page duplicate cluster and its fix, robots.txt vs noindex, SPA soft 404s, and how to check a URL as a crawler. |
| [wallet-key-migration.md](wallet-key-migration.md) | Incident record: stranded pool-agent wallets after the WALLET_ENCRYPTION_KEY migration. Also the standing runbook for where production's credentials live in Secret Manager, how to publish a new version, and the owner-gated master-wallet rotation. |
| [stranded-wallets.md](stranded-wallets.md) | The standing owner decision on custodial wallets sealed by that rotation: measured list, why recovery is impossible, what crediting or writing off the customer balances costs, and the commands for each. |
| [forge-multiview-migration-handoff.md](forge-multiview-migration-handoff.md) | Historical hand-off for the forge multi-view migrations (June 2026); superseded by gcp-production.md. |
diff --git a/docs/ops/search-indexing.md b/docs/ops/search-indexing.md
new file mode 100644
index 000000000..63a038f87
--- /dev/null
+++ b/docs/ops/search-indexing.md
@@ -0,0 +1,234 @@
+# Search indexing
+
+How three.ws pages reach a search index, what broke, and what to check before
+concluding a page "just isn't ranking yet". Companion to
+[seo-keyword-plan.md](./seo-keyword-plan.md), which covers what to write;
+this doc covers whether a crawler can read it at all.
+
+---
+
+## The three ways a page is served to a crawler
+
+Every public URL falls into one of these, and the failure modes are different:
+
+1. **A static page** (`pages/*.html`, most of `data/pages.json`). The HTML on
+ disk is the HTML a crawler sees. `scripts/inject-seo-meta.mjs` backfills the
+ title, description, canonical, OG/Twitter tags and JSON-LD from
+ `data/pages.json`, so the meta a crawler reads matches the catalog. Run it
+ with `--write` after adding a page; it never overwrites a tag a page already
+ has.
+
+2. **A server-rendered page** (`api/*-og.js`, `api/markets/*`). The handler
+ composes the document per request from the database. `/markets/news/...` and
+ the crawler pages below are here.
+
+3. **A single-page-app shell.** One HTML file backs an unbounded URL space:
+ `/coin/:id` renders `pages/coin.html`, `/m/:id` renders `pages/model.html`,
+ and so on. The shell has no content and no canonical until the controller
+ fills it in from an API call. Google indexes the *rendered* DOM, so this
+ works, but only if the controller sets the head tags. `src/seo-meta.js`
+ exists for exactly that (`setCanonical`, `markNoindex`).
+
+`/avatars/:id` and `/agents/:id` are the interesting case: a browser gets the
+SPA shell, and a crawler UA is rewritten by `vercel.json` to a server-rendered
+page instead (`api/avatar-detail-og.js`, `api/agent-detail-og.js`). Those two
+URL spaces are also the largest on the site: the avatars sub-sitemap alone
+submits up to 45,000 URLs.
+
+---
+
+## The duplicate-cluster failure (2026-09)
+
+Search Console, page indexing report, snapshot 2026-09-03: **30.2k indexed,
+25.1k not indexed**, and the single largest exclusion was **18,768 pages under
+"Duplicate without user-selected canonical"**, alongside 1,101 "Page with
+redirect", 1,382 "Crawled - currently not indexed" and 813 "Duplicate, Google
+chose different canonical than user".
+
+The cause was in the crawler pages, and it is worth understanding because the
+same shape is easy to reintroduce. The rendered page carried the avatar's name
+and description in `<meta>` tags, and the visible document was:
+
+```html
+<noscript>
+ <div class="shell"><h1>Shaw</h1><p>Rigged, walk-ready avatar…</p></div>
+</noscript>
+<div class="shell"><div class="spinner"></div><p>Loading Shaw…</p></div>
+<script>(function(){window.location.replace("/avatars/19da5ca4-…");})()</script>
+```
+
+Two things go wrong, and they compound:
+
+- **The redirect target is the URL the crawler is already on.** The rewrite is
+ keyed on the User-Agent, not the path, so re-requesting the same URL behind
+ the same UA returns a byte-identical response and the navigation repeats. A
+ crawler that runs JS (Googlebot, bingbot and Applebot all render before
+ indexing) never lands anywhere else, and may bucket the URL as a redirect.
+- **Rendering removes `<noscript>`.** The only real text on the page lived
+ inside it, so the document that reached the indexer was a spinner over the
+ word "Loading". Every avatar URL rendered to the same nothing.
+
+45,000 URLs with no distinguishing content is one duplicate cluster 45,000 wide.
+The canonical tag in the head could not save it: a cluster where nothing
+differs has no preferred version to point at.
+
+### The fix
+
+`api/_lib/crawler-page.js` now renders both crawler pages, and:
+
+- **The content is in the body.** Name, description, the 1200x630 card as a real
+ `<img>`, a definition list of facts (creator, category, rig state, fork count,
+ publication date for an avatar; skills, ERC-8004 identity, home URL for an
+ agent), tag chips linking into `/gallery` or `/agents`, and a footer of
+ related surfaces. Plus `3DModel` / `SoftwareApplication` JSON-LD and a
+ `BreadcrumbList`. No `<noscript>`, because nothing hides in it any more.
+- **The self-redirect is emitted only for crawlers that never index.**
+ `isSearchCrawler()` splits indexing crawlers (Googlebot, bingbot, Applebot,
+ GPTBot, ClaudeBot, PerplexityBot, and the rest) from link unfurlers
+ (Twitterbot, Slackbot, Embedly, Iframely). An unfurler still gets bounced to
+ the app, which is the friendly outcome for a human arriving behind a scraper
+ UA. A search engine gets content and no script.
+- **A missing entity 404s** instead of 302-ing to `/gallery` or `/agents`. Those
+ redirects were the "Page with redirect" and "Soft 404" rows: a crawler asked
+ about a deleted avatar and was told, in effect, "here is a different page".
+ Now it is told the truth, on a `noindex` document, and the URL leaves the
+ index.
+- **An id that names the other store 301s across.** Agent and avatar UUIDs are
+ separate spaces and links cross over (`resolveEntity` in `src/avatar-page.js`
+ does the same thing client-side), so `/agents/<an avatar id>` sends a real 301
+ to `/avatars/<id>` rather than reporting nothing.
+- **A database failure returns 503, not 404.** A read failure is ours, not the
+ URL's; a 404 there would drop live pages out of the index.
+- **Private agents are no longer served at all.** The agent query filtered only
+ on `deleted_at`, so an agent with `is_public = false` had its name and
+ description rendered to anything sending a bot UA. It now matches the
+ sitemap's predicate (`is_public = true`).
+
+### Content parity, not cloaking
+
+Serving crawlers a different document than browsers is dynamic serving, which is
+allowed as long as the content is equivalent. It is: the crawler page carries
+the same name, description, imagery, tags, and creator the interactive page
+shows. What it omits is the WebGL viewport a crawler cannot run. Keep it that
+way. If a fact appears on the crawler page that a visitor cannot find on the
+real one, that is the line.
+
+---
+
+## The other duplicate source: untouched onboarding rows
+
+Rendering real content only helps if the entities differ. Measured against
+production on 2026-09-07:
+
+| | Public | Indexable | Dropped |
+|---|---|---|---|
+| Agents | 3,397 | 2,038 | 1,359 (40%) |
+| Avatars | 67,164 | 66,994 | 170 |
+
+**1,163 of the 3,397 public agents were still named "My First Agent"**, and
+1,158 of those still carried the starter description word for word. Those pages
+are not thin, they are the same page with a different UUID, and no server-side
+rendering can separate them.
+
+[`api/_lib/indexable-entity.js`](../../api/_lib/indexable-entity.js) decides.
+An agent is indexable unless it keeps *both* its default name and its starter
+description; an avatar is indexable unless it keeps a default name with no prose
+at all (a generated avatar still gets its own description and tag set, which is
+why so few are dropped). Both the sitemap builders and the crawler pages import
+the predicate, so the file and the page can never disagree.
+
+A non-indexable entity is not hidden: its page still renders, still unfurls with
+its own card, and still links onward. It is served `noindex, follow` and left
+out of the sitemap. The moment its owner types a name or a description it
+becomes indexable again, with no migration and no backfill, which is why the
+rule reads the row instead of storing a flag.
+
+Re-measure before quoting these numbers:
+
+```bash
+node --input-type=module -e "
+import { config } from 'dotenv'; config({ path: '.env.local', quiet: true });
+const { sql } = await import('./api/_lib/db.js');
+const { isIndexableAgent } = await import('./api/_lib/indexable-entity.js');
+const rows = await sql\`select name, description from agent_identities
+ where deleted_at is null and is_public = true\`;
+console.log(rows.length, '->', rows.filter(isIndexableAgent).length);
+process.exit(0);"
+```
+
+---
+
+## Indexed, though blocked by robots.txt
+
+25 URLs sat in this state, and it is always the same mistake:
+`public/robots.txt` disallowed `/dashboard`, `/settings` and `/my-agents`, while
+every page under them already carried `<meta name="robots" content="noindex">`.
+A crawler that is not allowed to fetch the page never sees the noindex, so the
+URL stays in the index as a bare link. The two directives cancel each other out.
+
+Crawling is what removes them, so those three prefixes are now open in
+robots.txt and the `noindex` does the work. `pages/dashboard-next/data-api.html`,
+`developers.html` and `billing.html` were missing the tag and now carry it.
+
+**Rule: pick one.** `Disallow` keeps a page out of the crawl; `noindex` keeps it
+out of the index. Never both on the same URL.
+
+---
+
+## Soft 404s from SPA shells
+
+A shell that renders "not found" over a `200` is a soft 404. The router cannot
+know the entity is missing, but the controller can, so it calls `markNoindex()`
+from [`src/seo-meta.js`](../../src/seo-meta.js) at the moment it renders the
+empty state. Wired into `src/coin-page.js`, `src/model-page.js` and
+`src/signal-detail.js`. Any new shell with a not-found branch should do the
+same, and should call `setCanonical()` once it knows which entity it is showing.
+
+---
+
+## Checking a page yourself
+
+Fetch as a crawler rather than trusting a browser view, because the UA branch
+changes the response:
+
+```bash
+UA='Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/128.0.0.0 Safari/537.36'
+curl -sS -A "$UA" https://three.ws/avatars/<id> | grep -E 'canonical|robots|<h1>'
+```
+
+What a healthy indexable page looks like:
+
+- `200`, exactly one `<link rel="canonical">`, pointing at itself.
+- `<meta name="robots">` absent or `index, follow`.
+- A `<h1>` and body copy that differ from every sibling URL.
+- No `location.replace` anywhere in the document.
+
+Sub-sitemap sizes, which is the fastest way to see how much of each space is
+being submitted:
+
+```bash
+for t in core agents avatars widgets profiles news; do
+ echo "$t: $(curl -sS https://three.ws/sitemap/$t.xml | grep -c '<loc>')"
+done
+```
+
+`npm run audit:web` sweeps every page in `data/pages.json` in a real browser
+(see [page-audit.md](./page-audit.md)); it catches console errors, not indexing
+state, so run both.
+
+---
+
+## What to expect after a fix lands
+
+Search Console does not recount on deploy. Google has to recrawl each URL, and
+at this volume that is weeks, not days. The order things move in:
+
+1. "Page with redirect" and "Soft 404" fall first: they are decided by the
+ response, which changes immediately.
+2. "Duplicate without user-selected canonical" drains as pages are re-rendered
+ and start to differ from one another.
+3. Indexed count rises last.
+
+Use the URL Inspection tool on a handful of representative URLs to confirm the
+rendered HTML is what this doc describes before reading anything into the
+aggregate charts.
diff --git a/docs/ops/seo-keyword-plan.md b/docs/ops/seo-keyword-plan.md
index 1178030d9..53f3bfa9c 100644
--- a/docs/ops/seo-keyword-plan.md
+++ b/docs/ops/seo-keyword-plan.md
@@ -2,7 +2,7 @@
Verified keyword landscape and content calendar for three.ws across its product clusters: 3D generation, AI agents/MCP, Solana/x402/crypto, and enterprise AI (watsonx/Granite). Built 2026-07-17 from a deep-research pass (24 sources fetched, 25 claims adversarially verified, 23 confirmed) plus a live audit of the IBM Community publishing surface. Every SERP observation is dated 2026-07-17; re-verify before acting on this doc more than a quarter later.
-Related: [page-audit.md](./page-audit.md) covers on-site quality; this doc covers what to write and where to publish it.
+Related: [page-audit.md](./page-audit.md) covers on-site quality and [search-indexing.md](./search-indexing.md) covers whether a crawler can read a page at all; this doc covers what to write and where to publish it.
---
diff --git a/pages/agent-mind.html b/pages/agent-mind.html
index e1d5fdd26..639eb5644 100644
--- a/pages/agent-mind.html
+++ b/pages/agent-mind.html
@@ -6,6 +6,10 @@
<title data-i18n="agent_mind.meta_title">Mind Palace — three.ws</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<meta name="theme-color" content="#06080f" />
+ <!-- One agent, one indexable page: /agents/:id. This view is reachable at
+ both /agents/:id/mind and /agent/:id/mind and renders the same shell for
+ every agent, so indexing it would file thousands of identical URLs. -->
+ <meta name="robots" content="noindex, follow" />
<meta name="description" content="Walk through your agent's mind — every memory as a tangible object you can see, search, pin, and forget." data-i18n-attr="content:agent_mind.meta_description"/>
<meta property="og:type" content="website">
diff --git a/pages/dashboard-next/billing.html b/pages/dashboard-next/billing.html
index 585f83e0f..0b1562833 100644
--- a/pages/dashboard-next/billing.html
+++ b/pages/dashboard-next/billing.html
@@ -3,6 +3,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
+ <meta name="robots" content="noindex" />
<title data-i18n="dashboard_next_billing.meta_title">Billing &amp; Passes · Dashboard · three.ws</title>
<meta name="description" content="Manage your three.ws subscriptions and paid services: Premium pass status, on-chain Solana payment history, API keys minted by your passes, and the creator subscriptions you hold." data-i18n-attr="content:dashboard_next_billing.meta_description"/>
<link rel="icon" href="/favicon.ico" />
diff --git a/pages/dashboard-next/data-api.html b/pages/dashboard-next/data-api.html
index ad416ea0a..a49f03899 100644
--- a/pages/dashboard-next/data-api.html
+++ b/pages/dashboard-next/data-api.html
@@ -3,6 +3,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
+ <meta name="robots" content="noindex" />
<title data-i18n="dashboard_next_data_api.meta_title">Data API · Dashboard · three.ws</title>
<meta name="description" content="Developer console for the three.ws Data API: buy the Premium pass with $THREE, SOL, or USDC, manage your API key, watch usage, and copy working quickstarts." data-i18n-attr="content:dashboard_next_data_api.meta_description"/>
<link rel="icon" href="/favicon.ico" />
diff --git a/pages/dashboard-next/developers.html b/pages/dashboard-next/developers.html
index 0c612933f..fce7a2935 100644
--- a/pages/dashboard-next/developers.html
+++ b/pages/dashboard-next/developers.html
@@ -3,6 +3,7 @@
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
+ <meta name="robots" content="noindex" />
<title data-i18n="dashboard_next_developers.meta_title">Developer Hub · Dashboard · three.ws</title>
<meta name="description" content="Avatar Render API, webhooks, SDKs, and integration tools for developers building with three.ws avatars." data-i18n-attr="content:dashboard_next_developers.meta_description"/>
<link rel="icon" href="/favicon.ico" />
diff --git a/public/changelog.json b/public/changelog.json
index eb5eedcb5..64970f091 100644
--- a/public/changelog.json
+++ b/public/changelog.json
@@ -1,5 +1,5 @@
{
- "generated_at": "2026-09-07T05:35:32.758Z",
+ "generated_at": "2026-09-07T05:42:31.482Z",
"generated_by": "scripts/build-page-index.mjs from data/pages.json + data/changelog.json",
"site": {
"name": "three.ws",
diff --git a/public/features.json b/public/features.json
index 444ea7306..1228305d3 100644
--- a/public/features.json
+++ b/public/features.json
@@ -1,5 +1,5 @@
{
- "generated_at": "2026-09-07T05:35:32.744Z",
+ "generated_at": "2026-09-07T05:42:31.471Z",
"generated_by": "scripts/build-page-index.mjs from data/pages.json",
"site": {
"name": "three.ws",
diff --git a/public/robots.txt b/public/robots.txt
index f2c0840d8..4f3bc6336 100644
--- a/public/robots.txt
+++ b/public/robots.txt
@@ -1,11 +1,15 @@
User-agent: *
Allow: /
-Disallow: /my-agents
-Disallow: /dashboard
-Disallow: /settings
Disallow: /api/
Disallow: /studio-deposit-harness*
+# /dashboard, /settings and /my-agents are signed-in surfaces and every one of
+# their pages carries <meta name="robots" content="noindex">. A Disallow here
+# would hide that tag from the crawler that needs to read it: Google kept them
+# in the index as bare URLs ("Indexed, though blocked by robots.txt") precisely
+# because it was never allowed to fetch the page and see the noindex. Crawling
+# is what removes them, so leave these open.
+
# Embed routes serve raw runtime, not human content; keep them out of SERPs.
Disallow: /agent/*/embed
Disallow: /widget/*/embed
diff --git a/src/coin-page.js b/src/coin-page.js
index 9e0f7e70c..a9008f14e 100644
--- a/src/coin-page.js
+++ b/src/coin-page.js
@@ -18,6 +18,7 @@ import {
timeAgo,
escapeHtml as esc,
} from './shared/coin-format.js';
+import { markNoindex } from './seo-meta.js';
const TIME_RANGES = [
{ label: '24H', days: 1 },
@@ -1046,6 +1047,8 @@ async function loadMarkets(coin) {
// ── Not found / error states ────────────────────────────────────────────────
function renderNotFound(id) {
+ // A 200 with "not found" in the body is a soft 404 to a crawler. Say so.
+ markNoindex();
const mintish = MINT_RE.test(id);
$('cv-head').innerHTML = `
<h1 class="cv-h1">Coin not found</h1>
diff --git a/src/model-page.js b/src/model-page.js
index 1a75047c6..25a78be46 100644
--- a/src/model-page.js
+++ b/src/model-page.js
@@ -24,6 +24,7 @@ import {
embedSnippet,
chipsFor,
} from './model-lib.js';
+import { markNoindex, setCanonical } from './seo-meta.js';
const log = createLogger('model-page');
const $ = (id) => document.getElementById(id);
@@ -131,8 +132,7 @@ function patchMeta() {
set('meta[property="og:url"]', `https://three.ws/m/${c.id}`);
set('meta[property="og:image"]', `https://three.ws/api/forge/${c.id}/og`);
set('meta[name="twitter:image"]', `https://three.ws/api/forge/${c.id}/og`);
- const canonical = document.querySelector('link[rel="canonical"]');
- if (canonical) canonical.setAttribute('href', `https://three.ws/m/${c.id}`);
+ setCanonical(`/m/${c.id}`);
}
// ── viewer hero ──────────────────────────────────────────────────────────────
@@ -764,6 +764,8 @@ function renderSuggested() {
// ── error states ─────────────────────────────────────────────────────────────
function renderMissing(msg) {
+ // A 200 with "not found" in the body is a soft 404 to a crawler. Say so.
+ markNoindex();
const shell = $('mp-shell');
if (!shell) return;
shell.setAttribute('aria-busy', 'false');
diff --git a/src/seo-meta.js b/src/seo-meta.js
new file mode 100644
index 000000000..30cbfcba1
--- /dev/null
+++ b/src/seo-meta.js
@@ -0,0 +1,51 @@
+// Crawler-facing <head> state for pages the router builds from a template.
+//
+// One HTML file backs an unbounded URL space on this site: /coin/:id, /m/:id,
+// /signals/:id and friends all render the same shell and fill it in from an API
+// call. Two things follow, and neither is optional if the page should rank.
+//
+// 1. A canonical baked into the file would name the template, not the page,
+// so the shell ships without one and the controller sets a self-referential
+// canonical once it knows which entity it is showing. Google reads the
+// rendered DOM, so a canonical written here counts.
+//
+// 2. When the entity does not exist, the shell still answers 200 with an
+// apology in the body: a soft 404. The router cannot know, but the
+// controller does, so it marks the page noindex the moment it renders that
+// state. The URL then leaves the index as "excluded by noindex" instead of
+// accumulating as "soft 404".
+
+const ORIGIN = 'https://three.ws';
+
+/**
+ * Point this page's canonical at `path` (default: the URL being viewed, minus
+ * query and hash). Creates the tag when the shell shipped without one.
+ *
+ * @param {string} [path] absolute path, leading slash included
+ */
+export function setCanonical(path = location.pathname) {
+ let link = document.querySelector('link[rel="canonical"]');
+ if (!link) {
+ link = document.createElement('link');
+ link.rel = 'canonical';
+ document.head.appendChild(link);
+ }
+ link.href = `${ORIGIN}${path}`;
+}
+
+/**
+ * Declare this URL unindexable, for a view that renders "not found" over a 200.
+ * `follow` is kept so the links out of the empty state still carry the crawler
+ * somewhere useful. Also drops any canonical the page set earlier: a canonical
+ * on a noindex page only muddies which URL the directive applies to.
+ */
+export function markNoindex() {
+ let meta = document.querySelector('meta[name="robots"]');
+ if (!meta) {
+ meta = document.createElement('meta');
+ meta.name = 'robots';
+ document.head.appendChild(meta);
+ }
+ meta.setAttribute('content', 'noindex, follow');
+ document.querySelector('link[rel="canonical"]')?.remove();
+}
diff --git a/src/signal-detail.js b/src/signal-detail.js
index 266dd01d1..c97b8e276 100644
--- a/src/signal-detail.js
+++ b/src/signal-detail.js
@@ -10,6 +10,7 @@
import { apiFetch } from './api.js';
import { escapeHtml, fmtPct, compact, identicon } from './trader-format.js';
+import { markNoindex } from './seo-meta.js';
const $ = (s, r = document) => r.querySelector(s);
@@ -257,6 +258,8 @@ function wireSubscribe() {
}
function errorState(msg) {
+ // A 200 with "not found" in the body is a soft 404 to a crawler. Say so.
+ markNoindex();
const root = $('#sd-root');
root.setAttribute('aria-busy', 'false');
root.innerHTML = `<div class="sm-empty"><h1 id="sd-title" class="sd-error-title">${escapeHtml(msg)}</h1><p>This feed may have been paused, made unlisted, or never existed.</p><div class="sm-empty-actions"><a class="sm-cta" href="/signals">Browse all feeds →</a></div></div>`;
diff --git a/tests/api/agent-detail-og.test.js b/tests/api/agent-detail-og.test.js
new file mode 100644
index 000000000..e1258ef1b
--- /dev/null
+++ b/tests/api/agent-detail-og.test.js
@@ -0,0 +1,137 @@
+// The bot-UA rewrite for /agents/:id lands crawlers on api/agent-detail-og.js.
+// That page is the only version of the agent a search engine ever sees, so it
+// carries the agent's real content, keeps private agents out entirely (the
+// query used to omit is_public and served their name and description to
+// anything sending a bot UA), and never hands an indexing crawler a script that
+// navigates back to the URL it is already on.
+
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { makeReq, makeRes } from '../_helpers/monetization.js';
+
+const sqlState = { rows: [], queue: null, calls: 0, seen: [] };
+
+vi.mock('../../api/_lib/db.js', () => {
+ const sql = vi.fn(async (strings) => {
+ sqlState.seen.push(strings.join(' ? '));
+ const rows = sqlState.queue ? (sqlState.queue[sqlState.calls] ?? []) : sqlState.rows;
+ sqlState.calls += 1;
+ return rows;
+ });
+ return { sql, isDbUnavailableError: () => false, isDbCapacityError: () => false };
+});
+
+const { default: handler } = await import('../../api/agent-detail-og.js');
+
+const AGENT_ID = '00000000-0000-4000-8000-0000000000a1';
+const GOOGLEBOT =
+ 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/128.0.0.0 Safari/537.36';
+const UNFURLER = 'Mozilla/5.0 (compatible; TelegramBot; like TwitterBot)';
+
+const AGENT_ROW = {
+ id: AGENT_ID,
+ name: 'Atlas',
+ description: 'A market-watching agent.',
+ skills: ['research', 'trading'],
+ home_url: 'https://example.com/atlas',
+ erc8004_agent_id: '4821',
+ chain_id: 8453,
+ created_at: new Date('2026-05-04T00:00:00Z'),
+ updated_at: new Date('2026-06-01T00:00:00Z'),
+};
+
+async function invokeHtml(reqOpts) {
+ const req = makeReq({ headers: { 'user-agent': GOOGLEBOT }, ...reqOpts });
+ const res = makeRes();
+ await handler(req, res);
+ return { status: res.statusCode, headers: res.headers, html: res.body };
+}
+
+beforeEach(() => {
+ sqlState.rows = [];
+ sqlState.queue = null;
+ sqlState.calls = 0;
+ sqlState.seen = [];
+});
+
+describe('agent-detail-og', () => {
+ it('rejects a non-GET method with 405 before touching the db', async () => {
+ const { status } = await invokeHtml({
+ method: 'POST',
+ url: `/api/agent-detail-og?id=${AGENT_ID}`,
+ });
+ expect(status).toBe(405);
+ expect(sqlState.calls).toBe(0);
+ });
+
+ it('only reads public agents', async () => {
+ sqlState.rows = [AGENT_ROW];
+ await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });
+ expect(sqlState.seen[0]).toMatch(/is_public = true/);
+ });
+
+ it('404s an unknown agent instead of redirecting to the directory', async () => {
+ const { status, html } = await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });
+ expect(status).toBe(404);
+ expect(html).toMatch(/name="robots" content="noindex/);
+ expect(sqlState.calls).toBe(2);
+ });
+
+ it('301s to /avatars/:id when the id names an avatar, not an agent', async () => {
+ sqlState.queue = [[], [{ id: AGENT_ID }]];
+ const { status, headers } = await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });
+ expect(status).toBe(301);
+ expect(headers.location).toBe(`https://three.ws/avatars/${AGENT_ID}`);
+ });
+
+ it('renders the agent as real indexable content', async () => {
+ sqlState.rows = [AGENT_ROW];
+ const { status, html } = await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });
+ expect(status).toBe(200);
+ expect(html).toMatch(/<h1>Atlas<\/h1>/);
+ expect(html).toContain('A market-watching agent.');
+ expect(html).toContain('research, trading');
+ expect(html).toContain('ERC-8004 agent #4821');
+ expect(html).toMatch(/rel="canonical" href="https:\/\/three\.ws\/agents\//);
+ expect(html).toContain('"@type":"SoftwareApplication"');
+ expect(html).not.toContain('<noscript>');
+ });
+
+ it('marks the agent home link nofollow so a crawler does not pass authority out', async () => {
+ sqlState.rows = [AGENT_ROW];
+ const { html } = await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });
+ expect(html).toMatch(/rel="nofollow noopener"/);
+ });
+
+ it('renders an untouched onboarding agent noindex, with no canonical', async () => {
+ sqlState.rows = [
+ {
+ ...AGENT_ROW,
+ name: 'My First Agent',
+ description:
+ 'A friendly starter agent. Edit the personality and attach a 3D avatar \u2014 it goes live immediately.',
+ },
+ ];
+ const { status, html } = await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });
+ // The page still renders and still unfurls; it just does not ask to rank.
+ expect(status).toBe(200);
+ expect(html).toMatch(/<h1>My First Agent<\/h1>/);
+ expect(html).toContain('og:image');
+ expect(html).toMatch(/name="robots" content="noindex, follow"/);
+ expect(html).not.toContain('rel="canonical"');
+ });
+
+ it('never emits a self-navigating script for an indexing crawler', async () => {
+ sqlState.rows = [AGENT_ROW];
+ const { html } = await invokeHtml({ url: `/api/agent-detail-og?id=${AGENT_ID}` });
+ expect(html).not.toContain('location.replace');
+ });
+
+ it('still bounces a link unfurler to the app', async () => {
+ sqlState.rows = [AGENT_ROW];
+ const { html } = await invokeHtml({
+ url: `/api/agent-detail-og?id=${AGENT_ID}`,
+ headers: { 'user-agent': UNFURLER },
+ });
+ expect(html).toContain(`location.replace("/agents/${AGENT_ID}")`);
+ });
+});
diff --git a/tests/api/avatar-detail-og.test.js b/tests/api/avatar-detail-og.test.js
index d23d72759..d9e35e4bc 100644
--- a/tests/api/avatar-detail-og.test.js
+++ b/tests/api/avatar-detail-og.test.js
@@ -1,18 +1,21 @@
-// /avatars/:id serves a static shell, so social crawlers used to unfurl every
-// shared avatar link with the generic site card. The bot-UA rewrite now lands
-// them on api/avatar-detail-og.js, which must render the avatar's real name,
-// description, and rendered-image URL - and must never leak anything for a
-// private or missing avatar (302 passthrough instead).
+// /avatars/:id serves a static shell, so a bot-UA rewrite lands crawlers on
+// api/avatar-detail-og.js instead. That page is the only version of the avatar
+// a search engine ever sees, so it has to carry the avatar's real content, must
+// never leak a private or deleted avatar, and must never hand an indexing
+// crawler a script that navigates back to the URL it is already on (that loop,
+// plus content that lived only inside <noscript>, is what collapsed 45k avatar
+// URLs into one duplicate cluster in Search Console).
import { describe, it, expect, beforeEach, vi } from 'vitest';
import { makeReq, makeRes } from '../_helpers/monetization.js';
-const sqlState = { rows: [], calls: 0 };
+const sqlState = { rows: [], queue: null, calls: 0 };
vi.mock('../../api/_lib/db.js', () => {
const sql = vi.fn(async () => {
+ const rows = sqlState.queue ? (sqlState.queue[sqlState.calls] ?? []) : sqlState.rows;
sqlState.calls += 1;
- return sqlState.rows;
+ return rows;
});
return { sql, isDbUnavailableError: () => false, isDbCapacityError: () => false };
});
@@ -20,9 +23,27 @@ vi.mock('../../api/_lib/db.js', () => {
const { default: handler } = await import('../../api/avatar-detail-og.js');
const AVATAR_ID = '00000000-0000-4000-8000-000000000042';
+const GOOGLEBOT =
+ 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/128.0.0.0 Safari/537.36';
+const UNFURLER = 'Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)';
+
+const AVATAR_ROW = {
+ id: AVATAR_ID,
+ name: 'Nova <Scout>',
+ description: 'A chrome scout.',
+ alt_text: null,
+ tags: ['scout', 'chrome'],
+ model_category: 'avatar',
+ view_count: 41,
+ fork_count: 3,
+ parent_avatar_id: null,
+ created_at: new Date('2026-05-04T00:00:00Z'),
+ updated_at: new Date('2026-06-01T00:00:00Z'),
+ owner_username: 'nirholas',
+};
async function invokeHtml(reqOpts) {
- const req = makeReq(reqOpts);
+ const req = makeReq({ headers: { 'user-agent': GOOGLEBOT }, ...reqOpts });
const res = makeRes();
await handler(req, res);
return { status: res.statusCode, headers: res.headers, html: res.body };
@@ -30,6 +51,7 @@ async function invokeHtml(reqOpts) {
beforeEach(() => {
sqlState.rows = [];
+ sqlState.queue = null;
sqlState.calls = 0;
});
@@ -43,48 +65,78 @@ describe('avatar-detail-og', () => {
expect(sqlState.calls).toBe(0);
});
- it('302-passthroughs a malformed id without querying', async () => {
- const { status, headers } = await invokeHtml({
- url: '/api/avatar-detail-og?id=not-a-uuid',
- });
- expect(status).toBe(302);
- expect(headers.location).toMatch(/\/gallery$/);
+ it('serves a noindex page for a malformed id without querying', async () => {
+ const { status, html } = await invokeHtml({ url: '/api/avatar-detail-og?id=not-a-uuid' });
+ expect(status).toBe(200);
+ expect(html).toMatch(/name="robots" content="noindex/);
expect(sqlState.calls).toBe(0);
});
- it('302-passthroughs an unknown avatar', async () => {
- const { status, headers } = await invokeHtml({
- url: `/api/avatar-detail-og?id=${AVATAR_ID}`,
- });
- expect(status).toBe(302);
- expect(headers.location).toMatch(/\/gallery$/);
- expect(sqlState.calls).toBe(1);
+ it('404s an unknown avatar instead of redirecting to the gallery', async () => {
+ const { status, html } = await invokeHtml({ url: `/api/avatar-detail-og?id=${AVATAR_ID}` });
+ expect(status).toBe(404);
+ expect(html).toMatch(/name="robots" content="noindex/);
+ // Avatar miss, then the cross-store probe for an agent with the same id.
+ expect(sqlState.calls).toBe(2);
});
- it('renders OG meta from the avatar record for a public avatar', async () => {
- sqlState.rows = [
- {
- id: AVATAR_ID,
- name: 'Nova <Scout>',
- description: 'A chrome scout.',
- tags: ['scout', 'chrome'],
- model_category: 'avatar',
- owner_username: 'nirholas',
- },
- ];
+ it('301s to /agents/:id when the id names an agent, not an avatar', async () => {
+ sqlState.queue = [[], [{ id: AVATAR_ID }]];
+ const { status, headers } = await invokeHtml({ url: `/api/avatar-detail-og?id=${AVATAR_ID}` });
+ expect(status).toBe(301);
+ expect(headers.location).toBe(`https://three.ws/agents/${AVATAR_ID}`);
+ });
+
+ it('renders the avatar as real indexable content, not a spinner', async () => {
+ sqlState.rows = [AVATAR_ROW];
const { status, headers, html } = await invokeHtml({
url: `/api/avatar-detail-og?id=${AVATAR_ID}`,
});
expect(status).toBe(200);
expect(headers['content-type']).toContain('text/html');
- // Name is escaped, description + byline + tags flow into og:description.
- expect(html).toContain('Nova &lt;Scout&gt;');
+ // The name and description are in the body, not only in <meta>/<noscript>.
+ expect(html).toMatch(/<h1>Nova &lt;Scout&gt;<\/h1>/);
expect(html).toContain('A chrome scout.');
expect(html).toContain('@nirholas');
expect(html).toContain(`/api/avatars/${AVATAR_ID}/og`);
- expect(html).toContain(`/avatars/${AVATAR_ID}`);
+ expect(html).toMatch(/rel="canonical" href="https:\/\/three\.ws\/avatars\//);
expect(html).toContain('og:image');
expect(html).toContain('twitter:card');
- expect(html).toMatch(/rel="canonical"/);
+ expect(html).not.toContain('<noscript>');
+ // Structured data and internal links a crawler can follow.
+ expect(html).toContain('application/ld+json');
+ expect(html).toContain('"@type":"3DModel"');
+ expect(html).toContain('BreadcrumbList');
+ expect(html).toContain('href="/gallery"');
+ });
+
+ it('renders a default-named, undescribed avatar noindex', async () => {
+ sqlState.rows = [{ ...AVATAR_ROW, name: 'Avatar', description: null, alt_text: null }];
+ const { status, html } = await invokeHtml({ url: `/api/avatar-detail-og?id=${AVATAR_ID}` });
+ expect(status).toBe(200);
+ expect(html).toMatch(/name="robots" content="noindex, follow"/);
+ expect(html).not.toContain('rel="canonical"');
+ });
+
+ it('never emits a self-navigating script for an indexing crawler', async () => {
+ sqlState.rows = [AVATAR_ROW];
+ const { html } = await invokeHtml({ url: `/api/avatar-detail-og?id=${AVATAR_ID}` });
+ expect(html).not.toContain('location.replace');
+ });
+
+ it('still bounces a link unfurler to the app', async () => {
+ sqlState.rows = [AVATAR_ROW];
+ const { html } = await invokeHtml({
+ url: `/api/avatar-detail-og?id=${AVATAR_ID}`,
+ headers: { 'user-agent': UNFURLER },
+ });
+ expect(html).toContain(`location.replace("/avatars/${AVATAR_ID}")`);
+ });
+
+ it('escapes the avatar name everywhere it lands in the document', async () => {
+ sqlState.rows = [{ ...AVATAR_ROW, name: '"><script>alert(1)</script>' }];
+ const { html } = await invokeHtml({ url: `/api/avatar-detail-og?id=${AVATAR_ID}` });
+ expect(html).not.toContain('<script>alert(1)</script>');
+ expect(html).toContain('&lt;script&gt;alert(1)&lt;/script&gt;');
});
});
diff --git a/tests/api/crawler-page.test.js b/tests/api/crawler-page.test.js
new file mode 100644
index 000000000..54db98aa0
--- /dev/null
+++ b/tests/api/crawler-page.test.js
@@ -0,0 +1,102 @@
+// The UA split in api/_lib/crawler-page.js decides whether a crawler page ends
+// with a script that navigates to the URL it is already on. Get it wrong in one
+// direction and an indexing crawler loops on a content-free response; get it
+// wrong in the other and a human who arrives behind a scraper UA is stranded on
+// a static page. Both halves are asserted here.
+
+import { describe, it, expect } from 'vitest';
+import { isSearchCrawler, renderCrawlerPage, renderCrawlerNotFound } from '../../api/_lib/crawler-page.js';
+
+const SEARCH = [
+ 'Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; Googlebot/2.1; +http://www.google.com/bot.html) Chrome/128.0.0.0 Safari/537.36',
+ 'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)',
+ 'Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)',
+ 'Mozilla/5.0 (Device; OS) AppleWebKit/537.36 (KHTML, like Gecko) Version/8.0 Safari/600.0 (Applebot/0.1)',
+ 'Mozilla/5.0 (compatible; GPTBot/1.2; +https://openai.com/gptbot)',
+ 'Mozilla/5.0 (compatible; ClaudeBot/1.0; [email protected])',
+ 'Mozilla/5.0 (compatible; PerplexityBot/1.0; +https://perplexity.ai/perplexitybot)',
+ 'Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)',
+ 'Mozilla/5.0 (compatible; DuckDuckBot-Https/1.1; https://duckduckgo.com/duckduckbot)',
+];
+
+const UNFURLERS = [
+ 'Twitterbot/1.0',
+ 'facebookexternalhit/1.1 (+http://www.facebook.com/externalhit_uatext.php)',
+ 'Slackbot-LinkExpanding 1.0 (+https://api.slack.com/robots)',
+ 'Mozilla/5.0 (compatible; Discordbot/2.0; +https://discordapp.com)',
+ 'TelegramBot (like TwitterBot)',
+ 'Mozilla/5.0 (compatible; Embedly/0.2; snap; +http://support.embed.ly/)',
+ 'Iframely/1.3.1 (+https://iframely.com/docs/about)',
+ 'WhatsApp/2.19.81 A',
+];
+
+const PAGE = {
+ title: 'Nova',
+ desc: 'A chrome scout.',
+ pageUrl: 'https://three.ws/avatars/abc',
+ ogImage: 'https://three.ws/api/avatars/abc/og',
+ origin: 'https://three.ws',
+ trail: [
+ { name: 'Home', path: '/' },
+ { name: 'Gallery', path: '/gallery' },
+ { name: 'Nova', path: '/avatars/abc' },
+ ],
+};
+
+describe('isSearchCrawler', () => {
+ it('recognises every indexing crawler we serve pages to', () => {
+ for (const ua of SEARCH) expect(isSearchCrawler(ua), ua).toBe(true);
+ });
+
+ it('does not claim link unfurlers index anything', () => {
+ for (const ua of UNFURLERS) expect(isSearchCrawler(ua), ua).toBe(false);
+ });
+
+ it('treats a missing user-agent as a non-indexer', () => {
+ expect(isSearchCrawler(undefined)).toBe(false);
+ expect(isSearchCrawler('')).toBe(false);
+ expect(isSearchCrawler(null)).toBe(false);
+ });
+});
+
+describe('renderCrawlerPage', () => {
+ it('emits the self-navigating script only when asked', () => {
+ expect(renderCrawlerPage({ ...PAGE, redirect: false })).not.toContain('location.replace');
+ expect(renderCrawlerPage({ ...PAGE, redirect: true })).toContain(
+ 'location.replace("/avatars/abc")',
+ );
+ });
+
+ it('puts the content in the body and the canonical in the head', () => {
+ const html = renderCrawlerPage(PAGE);
+ expect(html).toMatch(/<h1>Nova<\/h1>/);
+ expect(html).toContain('A chrome scout.');
+ expect(html).toContain('<link rel="canonical" href="https://three.ws/avatars/abc">');
+ expect(html).toMatch(/name="robots" content="index, follow/);
+ expect(html).toContain('BreadcrumbList');
+ });
+
+ it('closes the ld+json script against a name that carries markup', () => {
+ const html = renderCrawlerPage({
+ ...PAGE,
+ title: 'x</script><script>alert(1)</script>',
+ jsonLd: { '@type': '3DModel', name: 'x</script><script>alert(1)</script>' },
+ });
+ expect(html).not.toContain('</script><script>alert(1)');
+ expect(html).toContain('\\u003c/script');
+ });
+});
+
+describe('renderCrawlerNotFound', () => {
+ it('is always noindex and carries a way out', () => {
+ const html = renderCrawlerNotFound({
+ heading: 'Gone',
+ message: 'Nothing here.',
+ origin: 'https://three.ws',
+ actions: [{ label: 'Gallery', href: '/gallery' }],
+ });
+ expect(html).toMatch(/name="robots" content="noindex, follow"/);
+ expect(html).toContain('href="/gallery"');
+ expect(html).not.toContain('rel="canonical"');
+ });
+});
diff --git a/tests/api/indexable-entity.test.js b/tests/api/indexable-entity.test.js
new file mode 100644
index 000000000..c0a06bf14
--- /dev/null
+++ b/tests/api/indexable-entity.test.js
@@ -0,0 +1,64 @@
+// A third of the public agents on three.ws are onboarding rows nobody edited:
+// same name, same description, different uuid. They are duplicates of each
+// other by construction, so they stay out of the sitemap and their crawler page
+// is served noindex. This is the predicate that decides, and it is shared by
+// both so the two can never disagree.
+
+import { describe, it, expect } from 'vitest';
+import { isIndexableAgent, isIndexableAvatar } from '../../api/_lib/indexable-entity.js';
+
+// The exact copy onboarding writes, escaped so the dash glyph the database
+// stores does not have to be typed into this repo.
+const STARTER =
+ 'A friendly starter agent. Edit the personality and attach a 3D avatar \u2014 it goes live immediately.';
+
+describe('isIndexableAgent', () => {
+ it('rejects the untouched onboarding row', () => {
+ expect(isIndexableAgent({ name: 'My First Agent', description: STARTER })).toBe(false);
+ expect(isIndexableAgent({ name: 'my first agent', description: STARTER })).toBe(false);
+ expect(isIndexableAgent({ name: ' My First Agent ', description: STARTER })).toBe(false);
+ expect(isIndexableAgent({ name: 'My First Agent', description: null })).toBe(false);
+ expect(isIndexableAgent({ name: 'Untitled Agent', description: '' })).toBe(false);
+ });
+
+ it('accepts a row where the owner supplied either half', () => {
+ expect(isIndexableAgent({ name: 'Atlas', description: STARTER })).toBe(true);
+ expect(isIndexableAgent({ name: 'My First Agent', description: 'Reads Solana mempools.' })).toBe(
+ true,
+ );
+ expect(isIndexableAgent({ name: 'Atlas', description: 'Reads Solana mempools.' })).toBe(true);
+ });
+
+ it('rejects a row with no name at all', () => {
+ expect(isIndexableAgent({ name: '', description: 'Something real.' })).toBe(false);
+ expect(isIndexableAgent({ name: null, description: 'Something real.' })).toBe(false);
+ expect(isIndexableAgent({})).toBe(false);
+ expect(isIndexableAgent(null)).toBe(false);
+ });
+});
+
+describe('isIndexableAvatar', () => {
+ it('keeps a generated avatar: the description and tags still tell it apart', () => {
+ expect(
+ isIndexableAvatar({
+ name: 'Shaw',
+ description: 'Rigged, walk-ready avatar, forged on three.ws.',
+ }),
+ ).toBe(true);
+ // A distinctive name is enough on its own.
+ expect(isIndexableAvatar({ name: 'Roman Legionary', description: null })).toBe(true);
+ });
+
+ it('rejects a default name with nothing written about it', () => {
+ expect(isIndexableAvatar({ name: 'My First Agent', description: null, alt_text: null })).toBe(
+ false,
+ );
+ expect(isIndexableAvatar({ name: 'Avatar', description: '', alt_text: '' })).toBe(false);
+ expect(isIndexableAvatar({ name: 'Untitled', description: null })).toBe(false);
+ });
+
+ it('accepts a default name once someone describes it', () => {
+ expect(isIndexableAvatar({ name: 'Avatar', description: 'My studio mascot.' })).toBe(true);
+ expect(isIndexableAvatar({ name: 'Avatar', alt_text: 'A chrome fox in a raincoat.' })).toBe(true);
+ });
+});
diff --git a/tests/sitemap-type.test.js b/tests/sitemap-type.test.js
index 0809e4984..421c88c59 100644
--- a/tests/sitemap-type.test.js
+++ b/tests/sitemap-type.test.js
@@ -88,8 +88,14 @@ beforeEach(() => {
db.queries = [];
db.fail = null;
db.rows = new Map([
- ['agent_identities', [{ id: 'agent-1', updated_at: AT, created_at: AT }]],
- ['avatars', [{ id: 'avatar-1', updated_at: null, created_at: AT }]],
+ [
+ 'agent_identities',
+ [{ id: 'agent-1', name: 'Atlas', description: 'Watches markets.', updated_at: AT, created_at: AT }],
+ ],
+ [
+ 'avatars',
+ [{ id: 'avatar-1', name: 'Nova', description: 'A chrome scout.', alt_text: null, updated_at: null, created_at: AT }],
+ ],
['widgets', [{ id: 'widget-1', updated_at: AT, created_at: AT }]],
['users u', [{ username: 'realhuman', updated_at: AT, created_at: AT }]],
]);
@@ -141,6 +147,32 @@ describe('GET /sitemap/<type>.xml: the six real types', () => {
expect(res.body).toContain('<loc>https://three.ws/u/a&amp;b&lt;c&gt;d&quot;e&apos;f</loc>');
});
+ it('leaves untouched onboarding rows out of the agents and avatars files', async () => {
+ // Same name, same description, different uuid: nothing a crawler can use
+ // to tell one from the next, so neither belongs in a sitemap.
+ db.rows.set('agent_identities', [
+ { id: 'real-1', name: 'Atlas', description: 'Watches markets.', updated_at: AT, created_at: AT },
+ {
+ id: 'starter-1',
+ name: 'My First Agent',
+ description:
+ 'A friendly starter agent. Edit the personality and attach a 3D avatar \u2014 it goes live immediately.',
+ updated_at: AT,
+ created_at: AT,
+ },
+ ]);
+ db.rows.set('avatars', [
+ { id: 'real-2', name: 'Nova', description: 'A chrome scout.', alt_text: null, updated_at: AT, created_at: AT },
+ { id: 'blank-2', name: 'Avatar', description: null, alt_text: null, updated_at: AT, created_at: AT },
+ ]);
+
+ const agents = locs((await call('agents')).body);
+ expect(agents).toEqual(['https://three.ws/agents/real-1']);
+
+ const avatars = locs((await call('avatars')).body);
+ expect(avatars).toEqual(['https://three.ws/avatars/real-2']);
+ });
+
it('drains the news archive newest month first', async () => {
const found = locs((await call('news')).body);
expect(found).toEqual([