Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(portal): honour robots.txt on every build, with a test that proves it

nirholasd3ba8a0c739e
diff --git a/packages/portal/example/index.html b/packages/portal/example/index.html
new file mode 100644
index 000000000..c528b1748
--- /dev/null
+++ b/packages/portal/example/index.html
@@ -0,0 +1,78 @@
+<!doctype html>
+<html lang="en">
+ <head>
+ <meta charset="utf-8" />
+ <meta name="viewport" content="width=device-width,initial-scale=1" />
+ <title>@three-ws/portal example</title>
+ <style>
+ :root { color-scheme: dark; }
+ body {
+ margin: 0;
+ min-height: 100vh;
+ background: #07080c;
+ color: #e8e8e8;
+ font: 16px/1.6 system-ui, sans-serif;
+ display: grid;
+ place-items: center;
+ padding: 24px;
+ }
+ main { width: min(880px, 100%); }
+ h1 { font-size: 1.6rem; margin: 0 0 8px; }
+ p { color: #9aa0aa; margin: 0 0 20px; }
+ form { display: flex; gap: 8px; margin-bottom: 20px; }
+ input {
+ flex: 1; padding: 12px; border-radius: 10px; border: 1px solid #2a2d36;
+ background: #10121a; color: inherit; font-family: ui-monospace, monospace;
+ }
+ button { padding: 12px 18px; border-radius: 10px; border: 0; background: #fff; color: #111; font-weight: 600; cursor: pointer; }
+ #summary { font-family: ui-monospace, monospace; font-size: 13px; color: #9aa0aa; margin-bottom: 16px; white-space: pre-wrap; }
+ #stage { min-height: 520px; }
+ </style>
+ </head>
+ <body>
+ <main>
+ <h1>Walk a website</h1>
+ <p>Type an address. The summary comes from the SDK; the world below it is the live renderer.</p>
+ <form id="form">
+ <label class="sr-only" for="url">Website address</label>
+ <input id="url" value="example.com" spellcheck="false" />
+ <button type="submit">Build it</button>
+ </form>
+ <div id="summary">Nothing built yet.</div>
+ <div id="stage"></div>
+ </main>
+
+ <script type="module">
+ // From npm: import { fetchWorld, describeWorld, mountPortal } from '@three-ws/portal';
+ import { fetchWorld, describeWorld, mountPortal } from '../src/index.js';
+
+ const form = document.getElementById('form');
+ const input = document.getElementById('url');
+ const summary = document.getElementById('summary');
+ const stage = document.getElementById('stage');
+
+ async function build(url) {
+ summary.textContent = `Building ${url}…`;
+ stage.replaceChildren();
+ try {
+ const { world, cached } = await fetchWorld(url);
+ const d = describeWorld(world);
+ summary.textContent = [
+ `${d.title}`,
+ `${d.sections} districts · ${d.words} words · ${d.doors} doors · ${d.billboards} billboards${cached ? ' · cached' : ''}`,
+ d.tallest ? `tallest: ${d.tallest.name} (${d.tallest.words} words)` : '',
+ ].filter(Boolean).join('\n');
+ mountPortal(stage, { url, height: 520 });
+ } catch (err) {
+ summary.textContent = `${err.code || 'error'}: ${err.message}`;
+ }
+ }
+
+ form.addEventListener('submit', (e) => {
+ e.preventDefault();
+ build(input.value.trim());
+ });
+ build(input.value);
+ </script>
+ </body>
+</html>
diff --git a/tests/portal-robots.test.js b/tests/portal-robots.test.js
new file mode 100644
index 000000000..8adb97b2f
--- /dev/null
+++ b/tests/portal-robots.test.js
@@ -0,0 +1,85 @@
+// The robots matcher is the part of Portal that decides whether we are allowed
+// to read a page at all, so it is tested against the cases RFC 9309 actually
+// specifies rather than the ones that felt obvious.
+import { describe, it, expect } from 'vitest';
+import { parseRobots, matchRule, isAllowed } from '../api/_lib/portal/robots.js';
+
+const UA = 'threewsportalbot';
+
+describe('parseRobots', () => {
+ it('groups consecutive user-agent lines together', () => {
+ const groups = parseRobots(`
+User-agent: a
+User-agent: b
+Disallow: /x
+
+User-agent: c
+Disallow: /y
+`);
+ expect(groups).toHaveLength(2);
+ expect(groups[0].agents).toEqual(['a', 'b']);
+ expect(groups[0].rules).toEqual([{ allow: false, path: '/x' }]);
+ expect(groups[1].agents).toEqual(['c']);
+ });
+
+ it('strips comments and ignores junk lines', () => {
+ const groups = parseRobots('# hello\nUser-agent: * # everyone\nDisallow: /private # secret\nnonsense\n');
+ expect(groups[0].rules).toEqual([{ allow: false, path: '/private' }]);
+ });
+
+ it('treats an empty Disallow as no rule at all', () => {
+ expect(parseRobots('User-agent: *\nDisallow:')[0].rules).toEqual([]);
+ });
+});
+
+describe('matchRule', () => {
+ it('matches a prefix', () => {
+ expect(matchRule('/docs', '/docs/page')).toBeGreaterThan(0);
+ expect(matchRule('/docs', '/blog')).toBe(-1);
+ });
+
+ it('handles * wildcards anywhere in the pattern', () => {
+ expect(matchRule('/*/private', '/a/private')).toBeGreaterThan(0);
+ expect(matchRule('/*.pdf', '/files/report.pdf')).toBeGreaterThan(0);
+ expect(matchRule('/*.pdf', '/files/report.html')).toBe(-1);
+ });
+
+ it('anchors with a trailing $', () => {
+ expect(matchRule('/page$', '/page')).toBeGreaterThan(0);
+ expect(matchRule('/page$', '/page/more')).toBe(-1);
+ });
+});
+
+describe('isAllowed', () => {
+ it('allows everything when robots.txt is missing or empty', () => {
+ expect(isAllowed(null, '/anything', UA)).toBe(true);
+ expect(isAllowed('', '/anything', UA)).toBe(true);
+ });
+
+ it('honours a wildcard group', () => {
+ const body = 'User-agent: *\nDisallow: /private\n';
+ expect(isAllowed(body, '/private/thing', UA)).toBe(false);
+ expect(isAllowed(body, '/public', UA)).toBe(true);
+ });
+
+ it('prefers the most specific matching user-agent group', () => {
+ const body = 'User-agent: *\nDisallow: /\n\nUser-agent: ThreeWSPortalBot\nAllow: /\n';
+ expect(isAllowed(body, '/anything', UA)).toBe(true);
+ expect(isAllowed(body, '/anything', 'someotherbot')).toBe(false);
+ });
+
+ it('lets the longest matching rule win, and Allow win a tie', () => {
+ const body = 'User-agent: *\nDisallow: /docs\nAllow: /docs/public\n';
+ expect(isAllowed(body, '/docs/secret', UA)).toBe(false);
+ expect(isAllowed(body, '/docs/public/page', UA)).toBe(true);
+
+ const tie = 'User-agent: *\nDisallow: /page\nAllow: /page\n';
+ expect(isAllowed(tie, '/page', UA)).toBe(true);
+ });
+
+ it('reads the query string as part of the path, the way a crawler sends it', () => {
+ const body = 'User-agent: *\nDisallow: /*?print=1\n';
+ expect(isAllowed(body, '/article?print=1', UA)).toBe(false);
+ expect(isAllowed(body, '/article', UA)).toBe(true);
+ });
+});