Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(portal): honour robots.txt on every build, with a test that proves it
nirholasd3ba8a0c739e
diff --git a/packages/portal/example/index.html b/packages/portal/example/index.htmlnew file mode 100644index 000000000..c528b1748--- /dev/null+++ b/packages/portal/example/index.html@@ -0,0 +1,78 @@+<!doctype html>+<html lang="en">+ <head>+ <meta charset="utf-8" />+ <meta name="viewport" content="width=device-width,initial-scale=1" />+ <title>@three-ws/portal example</title>+ <style>+ :root { color-scheme: dark; }+ body {+ margin: 0;+ min-height: 100vh;+ background: #07080c;+ color: #e8e8e8;+ font: 16px/1.6 system-ui, sans-serif;+ display: grid;+ place-items: center;+ padding: 24px;+ }+ main { width: min(880px, 100%); }+ h1 { font-size: 1.6rem; margin: 0 0 8px; }+ p { color: #9aa0aa; margin: 0 0 20px; }+ form { display: flex; gap: 8px; margin-bottom: 20px; }+ input {+ flex: 1; padding: 12px; border-radius: 10px; border: 1px solid #2a2d36;+ background: #10121a; color: inherit; font-family: ui-monospace, monospace;+ }+ button { padding: 12px 18px; border-radius: 10px; border: 0; background: #fff; color: #111; font-weight: 600; cursor: pointer; }+ #summary { font-family: ui-monospace, monospace; font-size: 13px; color: #9aa0aa; margin-bottom: 16px; white-space: pre-wrap; }+ #stage { min-height: 520px; }+ </style>+ </head>+ <body>+ <main>+ <h1>Walk a website</h1>+ <p>Type an address. The summary comes from the SDK; the world below it is the live renderer.</p>+ <form id="form">+ <label class="sr-only" for="url">Website address</label>+ <input id="url" value="example.com" spellcheck="false" />+ <button type="submit">Build it</button>+ </form>+ <div id="summary">Nothing built yet.</div>+ <div id="stage"></div>+ </main>++ <script type="module">+ // From npm: import { fetchWorld, describeWorld, mountPortal } from '@three-ws/portal';+ import { fetchWorld, describeWorld, mountPortal } from '../src/index.js';++ const form = document.getElementById('form');+ const input = document.getElementById('url');+ const summary = document.getElementById('summary');+ const stage = document.getElementById('stage');++ async function build(url) {+ summary.textContent = `Building ${url}…`;+ stage.replaceChildren();+ try {+ const { world, cached } = await fetchWorld(url);+ const d = describeWorld(world);+ summary.textContent = [+ `${d.title}`,+ `${d.sections} districts · ${d.words} words · ${d.doors} doors · ${d.billboards} billboards${cached ? ' · cached' : ''}`,+ d.tallest ? `tallest: ${d.tallest.name} (${d.tallest.words} words)` : '',+ ].filter(Boolean).join('\n');+ mountPortal(stage, { url, height: 520 });+ } catch (err) {+ summary.textContent = `${err.code || 'error'}: ${err.message}`;+ }+ }++ form.addEventListener('submit', (e) => {+ e.preventDefault();+ build(input.value.trim());+ });+ build(input.value);+ </script>+ </body>+</html>diff --git a/tests/portal-robots.test.js b/tests/portal-robots.test.jsnew file mode 100644index 000000000..8adb97b2f--- /dev/null+++ b/tests/portal-robots.test.js@@ -0,0 +1,85 @@+// The robots matcher is the part of Portal that decides whether we are allowed+// to read a page at all, so it is tested against the cases RFC 9309 actually+// specifies rather than the ones that felt obvious.+import { describe, it, expect } from 'vitest';+import { parseRobots, matchRule, isAllowed } from '../api/_lib/portal/robots.js';++const UA = 'threewsportalbot';++describe('parseRobots', () => {+ it('groups consecutive user-agent lines together', () => {+ const groups = parseRobots(`+User-agent: a+User-agent: b+Disallow: /x++User-agent: c+Disallow: /y+`);+ expect(groups).toHaveLength(2);+ expect(groups[0].agents).toEqual(['a', 'b']);+ expect(groups[0].rules).toEqual([{ allow: false, path: '/x' }]);+ expect(groups[1].agents).toEqual(['c']);+ });++ it('strips comments and ignores junk lines', () => {+ const groups = parseRobots('# hello\nUser-agent: * # everyone\nDisallow: /private # secret\nnonsense\n');+ expect(groups[0].rules).toEqual([{ allow: false, path: '/private' }]);+ });++ it('treats an empty Disallow as no rule at all', () => {+ expect(parseRobots('User-agent: *\nDisallow:')[0].rules).toEqual([]);+ });+});++describe('matchRule', () => {+ it('matches a prefix', () => {+ expect(matchRule('/docs', '/docs/page')).toBeGreaterThan(0);+ expect(matchRule('/docs', '/blog')).toBe(-1);+ });++ it('handles * wildcards anywhere in the pattern', () => {+ expect(matchRule('/*/private', '/a/private')).toBeGreaterThan(0);+ expect(matchRule('/*.pdf', '/files/report.pdf')).toBeGreaterThan(0);+ expect(matchRule('/*.pdf', '/files/report.html')).toBe(-1);+ });++ it('anchors with a trailing $', () => {+ expect(matchRule('/page$', '/page')).toBeGreaterThan(0);+ expect(matchRule('/page$', '/page/more')).toBe(-1);+ });+});++describe('isAllowed', () => {+ it('allows everything when robots.txt is missing or empty', () => {+ expect(isAllowed(null, '/anything', UA)).toBe(true);+ expect(isAllowed('', '/anything', UA)).toBe(true);+ });++ it('honours a wildcard group', () => {+ const body = 'User-agent: *\nDisallow: /private\n';+ expect(isAllowed(body, '/private/thing', UA)).toBe(false);+ expect(isAllowed(body, '/public', UA)).toBe(true);+ });++ it('prefers the most specific matching user-agent group', () => {+ const body = 'User-agent: *\nDisallow: /\n\nUser-agent: ThreeWSPortalBot\nAllow: /\n';+ expect(isAllowed(body, '/anything', UA)).toBe(true);+ expect(isAllowed(body, '/anything', 'someotherbot')).toBe(false);+ });++ it('lets the longest matching rule win, and Allow win a tie', () => {+ const body = 'User-agent: *\nDisallow: /docs\nAllow: /docs/public\n';+ expect(isAllowed(body, '/docs/secret', UA)).toBe(false);+ expect(isAllowed(body, '/docs/public/page', UA)).toBe(true);++ const tie = 'User-agent: *\nDisallow: /page\nAllow: /page\n';+ expect(isAllowed(tie, '/page', UA)).toBe(true);+ });++ it('reads the query string as part of the path, the way a crawler sends it', () => {+ const body = 'User-agent: *\nDisallow: /*?print=1\n';+ expect(isAllowed(body, '/article?print=1', UA)).toBe(false);+ expect(isAllowed(body, '/article', UA)).toBe(true);+ });+});