Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(companion): let the corner avatar walk on and deliver a user's own important messages
nirholasd1f7b4595b8d
diff --git a/api/_lib/companion/crypto.js b/api/_lib/companion/crypto.jsnew file mode 100644index 000000000..d1332b2e5--- /dev/null+++ b/api/_lib/companion/crypto.js@@ -0,0 +1,71 @@+// Encrypt/decrypt a companion source's BYOK credentials at rest.+//+// Same AES-256-GCM + HKDF construction as api/_lib/provider-keys.js, with its+// own salt so a bot token can never be decrypted by a key derived for another+// key space. The plaintext is a small JSON blob (bot token, IMAP password, a+// calendar's secret ICS URL), so it round-trips as a string.++import { webcrypto } from 'node:crypto';+import { env } from '../env.js';++const subtle = globalThis.crypto?.subtle || webcrypto.subtle;++function randomBytes(n) {+ const b = new Uint8Array(n);+ (globalThis.crypto || webcrypto).getRandomValues(b);+ return b;+}++async function deriveKey() {+ const raw = new TextEncoder().encode(env.JWT_SECRET);+ const base = await subtle.importKey('raw', raw, 'HKDF', false, ['deriveKey']);+ return subtle.deriveKey(+ {+ name: 'HKDF',+ hash: 'SHA-256',+ salt: new TextEncoder().encode('companion-source-config-v1'),+ info: new Uint8Array(0),+ },+ base,+ { name: 'AES-GCM', length: 256 },+ false,+ ['encrypt', 'decrypt'],+ );+}++export async function encryptConfig(config) {+ const key = await deriveKey();+ const iv = randomBytes(12);+ const data = new TextEncoder().encode(JSON.stringify(config));+ const ct = await subtle.encrypt({ name: 'AES-GCM', iv }, key, data);+ const buf = new Uint8Array(iv.length + ct.byteLength);+ buf.set(iv, 0);+ buf.set(new Uint8Array(ct), iv.length);+ return Buffer.from(buf).toString('base64');+}++export async function decryptConfig(ciphertext) {+ const key = await deriveKey();+ const raw = Buffer.from(ciphertext, 'base64');+ const plain = await subtle.decrypt({ name: 'AES-GCM', iv: raw.subarray(0, 12) }, key, raw.subarray(12));+ return JSON.parse(new TextDecoder().decode(plain));+}++// What the UI is allowed to see about a stored credential: enough to recognise+// which account is connected, never enough to reuse it elsewhere.+export function redactConfig(kind, config) {+ if (!config || typeof config !== 'object') return {};+ if (kind === 'telegram') {+ const token = String(config.bot_token || '');+ return { bot_username: config.bot_username || null, bot_token_hint: token ? `${token.slice(0, 6)}…${token.slice(-4)}` : null };+ }+ if (kind === 'calendar') {+ let host = null;+ try { host = new URL(config.ics_url).host; } catch { host = null; }+ return { ics_host: host, lookahead_minutes: config.lookahead_minutes || 60 };+ }+ if (kind === 'email') {+ return { host: config.host || null, port: config.port || null, user: config.user || null, folder: config.folder || 'INBOX' };+ }+ return {};+}diff --git a/api/_lib/migrations/20260828140000_companion_harness.sql b/api/_lib/migrations/20260828140000_companion_harness.sqlnew file mode 100644index 000000000..3b9b27da7--- /dev/null+++ b/api/_lib/migrations/20260828140000_companion_harness.sql@@ -0,0 +1,108 @@+begin;++-- The Companion: a personal 3D assistant that delivers a user's own important+-- messages in person. Everything here is per-user and BYOK: the credentials for+-- each source live encrypted in companion_sources.config_encrypted (AES-256-GCM,+-- api/_lib/companion/crypto.js), never in plaintext columns.++create table if not exists companion_settings (+ user_id uuid primary key references users(id) on delete cascade,+ enabled boolean not null default true,+ -- Only events scoring at or above this get a spoken delivery (0-100).+ threshold integer not null default 60,+ -- Quiet hours in the user's own timezone; a delivery inside the window is+ -- stored and shown in the feed but never spoken or pushed.+ quiet_start smallint,+ quiet_end smallint,+ timezone text not null default 'UTC',+ -- Default stage presence when a sender has no contact-specific avatar.+ avatar_glb_url text,+ voice text not null default 'alloy',+ -- Bearer for the phone/desktop bridge (POST /api/companion/ingest).+ ingest_token text not null,+ push_enabled boolean not null default true,+ created_at timestamptz not null default now(),+ updated_at timestamptz not null default now(),+ constraint companion_settings_threshold_chk check (threshold between 0 and 100),+ constraint companion_settings_quiet_chk+ check ((quiet_start is null) = (quiet_end is null)+ and (quiet_start is null or (quiet_start between 0 and 23 and quiet_end between 0 and 23)))+);++create unique index if not exists companion_settings_ingest_token_uniq+ on companion_settings (ingest_token);++create table if not exists companion_sources (+ id uuid primary key default gen_random_uuid(),+ user_id uuid not null references users(id) on delete cascade,+ kind text not null,+ label text not null,+ config_encrypted text not null,+ -- Lane cursor: telegram update offset, imap uid, calendar seen-uid set.+ cursor jsonb not null default '{}'::jsonb,+ enabled boolean not null default true,+ status text not null default 'pending',+ last_error text,+ last_polled_at timestamptz,+ last_event_at timestamptz,+ created_at timestamptz not null default now(),+ constraint companion_sources_kind_chk check (kind in ('telegram', 'calendar', 'email')),+ constraint companion_sources_status_chk check (status in ('pending', 'ok', 'error'))+);++create index if not exists companion_sources_user_idx on companion_sources (user_id, kind);+create index if not exists companion_sources_poll_idx on companion_sources (enabled, last_polled_at);++-- Who the message is from, and who shows up to deliver it. One row per identity+-- the user cares about (a telegram @handle, an email address, a phone number).+create table if not exists companion_contacts (+ id uuid primary key default gen_random_uuid(),+ user_id uuid not null references users(id) on delete cascade,+ identifier text not null,+ display_name text not null,+ avatar_glb_url text,+ avatar_image_url text,+ voice text,+ -- Added to every event from this identity, so family outranks a newsletter.+ priority_boost integer not null default 0,+ created_at timestamptz not null default now(),+ constraint companion_contacts_boost_chk check (priority_boost between -100 and 100)+);++create unique index if not exists companion_contacts_identity_uniq+ on companion_contacts (user_id, lower(identifier));++create table if not exists companion_events (+ id uuid primary key default gen_random_uuid(),+ user_id uuid not null references users(id) on delete cascade,+ source_id uuid references companion_sources(id) on delete set null,+ source_kind text not null,+ -- Stable per-lane id (telegram message id, imap uid, calendar uid+start,+ -- bridge-supplied id) so a re-poll never speaks the same message twice.+ external_id text not null,+ contact_id uuid references companion_contacts(id) on delete set null,+ sender text,+ sender_id text,+ title text not null,+ body text,+ url text,+ importance integer not null default 0,+ reason text,+ spoken_line text,+ triage_engine text not null default 'rules',+ occurs_at timestamptz,+ delivered_at timestamptz,+ dismissed_at timestamptz,+ created_at timestamptz not null default now(),+ constraint companion_events_importance_chk check (importance between 0 and 100)+);++create unique index if not exists companion_events_dedupe_uniq+ on companion_events (user_id, source_kind, external_id);+create index if not exists companion_events_feed_idx+ on companion_events (user_id, created_at desc);+create index if not exists companion_events_pending_idx+ on companion_events (user_id, delivered_at, importance desc)+ where delivered_at is null and dismissed_at is null;++commit;diff --git a/api/_lib/notify-prefs.js b/api/_lib/notify-prefs.jsindex 128da1795..07ea0e309 100644--- a/api/_lib/notify-prefs.js+++ b/api/_lib/notify-prefs.js@@ -4,7 +4,8 @@// A notification has a `type` (e.g. 'skill_purchased'). Types roll up into a// small set of user-facing CATEGORIES. The user controls, per category, which// CHANNELS deliver it. Channels: in_app (the bell inbox), push (Web Push),-// email (Resend transactional), telegram (per-account bot alerts).+// email (Resend transactional), telegram (per-account bot alerts), avatar (the+// corner companion walks on and says it out loud, see src/notification-herald.js).//// Preferences are stored sparsely in notification_preferences.prefs as// { categories: { sales: { push: false }, ... }, telegram_chat_id: '123' }@@ -59,7 +60,7 @@ export const CATEGORIES = [},];-export const CHANNELS = ['in_app', 'push', 'email', 'telegram'];+export const CHANNELS = ['in_app', 'push', 'email', 'telegram', 'avatar'];/*** Channels that always deliver for a category, regardless of preferences.@@ -124,23 +125,39 @@ export function categoryForType(type) {return TYPE_CATEGORY[type] || 'account';}+/**+ * The whole type to category map, for clients that have to resolve a category+ * without a round trip per notification (the avatar channel runs in the+ * browser). Served by GET /api/notifications/preferences so the browser never+ * keeps its own copy of this table.+ */+export function typeCategoryMap() {+ return { ...TYPE_CATEGORY };+}+// Default channel matrix. in_app is on everywhere (the bell is the baseline).// push is on by default *for users who have subscribed a device* — a user with// no push subscription simply never receives one, so defaulting it on is not// spammy. email defaults on only for money + security; off for the higher-// frequency social/irl/alerts categories. telegram is always opt-in (needs a// linked chat id) so it defaults off.+// avatar is the most interruptive channel there is: the corner companion walks+// on screen and says the line out loud. It defaults on only for the three+// categories worth stopping the visitor for (money in, a creation that+// finished while they waited, account and security), and off for the+// higher-frequency ones. Every category stays togglable in the preference+// center either way.const DEFAULTS = {- sales: { in_app: true, push: true, email: true, telegram: false },- purchases: { in_app: true, push: true, email: true, telegram: false },- social: { in_app: true, push: true, email: false, telegram: false },- irl: { in_app: true, push: true, email: false, telegram: false },- alerts: { in_app: true, push: true, email: false, telegram: true },+ sales: { in_app: true, push: true, email: true, telegram: false, avatar: true },+ purchases: { in_app: true, push: true, email: true, telegram: false, avatar: false },+ social: { in_app: true, push: true, email: false, telegram: false, avatar: false },+ irl: { in_app: true, push: true, email: false, telegram: false, avatar: false },+ alerts: { in_app: true, push: true, email: false, telegram: true, avatar: false },// Only unattended completions notify (api/cron/forge-finalize.js), so email// defaulting on is the feature, not spam: the user left the page and asked// to hear back.- creations: { in_app: true, push: true, email: true, telegram: false },- account: { in_app: true, push: true, email: true, telegram: false },+ creations: { in_app: true, push: true, email: true, telegram: false, avatar: true },+ account: { in_app: true, push: true, email: true, telegram: false, avatar: true },};/** The full default matrix, used to seed the preference-center UI. */diff --git a/api/notifications/preferences.js b/api/notifications/preferences.jsindex 8b15e3658..1ecb5ab6e 100644--- a/api/notifications/preferences.js+++ b/api/notifications/preferences.js@@ -1,7 +1,8 @@// Unified notification preference center.//// GET /api/notifications/preferences-// → { categories: [...], channels: [...], prefs: {...}, push: {...} }+// → { categories: [...], channels: [...], type_categories: {...},+// prefs: {...}, push: {...} }// the full resolved matrix + metadata the UI renders from.// PUT /api/notifications/preferences { categories: {...}, telegram_chat_id }// → merge a sanitised sparse override onto what is already stored; unknown@@ -25,6 +26,7 @@ import {readStoredPrefs,resolvePrefs,sanitizePrefs,+ typeCategoryMap,} from '../_lib/notify-prefs.js';const putBody = z.object({@@ -55,6 +57,10 @@ export default wrap(async (req, res) => {return json(res, 200, {categories: CATEGORIES,channels: CHANNELS,+ // The avatar channel is delivered client-side (src/notification-herald.js),+ // so the browser needs the same type to category mapping the server+ // gates push and email with. Shipping it here keeps one source of truth.+ type_categories: typeCategoryMap(),prefs,push: { subscribed_devices: pushRow?.count ?? 0 },});diff --git a/package-lock.json b/package-lock.jsonindex 71c48b0f7..f158287ca 100644--- a/package-lock.json+++ b/package-lock.json@@ -168,6 +168,7 @@"gltf-validator": "^2.0.0-dev.3.10","helius-sdk": "^3.0.0","idb-keyval": "^6.3.0",+ "imapflow": "^1.7.6","jose": "^5.9.6","lightweight-charts": "^5.2.0","lil-gui": "^0.21.0",@@ -181,6 +182,7 @@"nanoid": "^3.3.11","nipplejs": "^1.0.4","node-html-parser": "^7.1.0",+ "node-ical": "^0.20.1","onnxruntime-web": "^1.22.0","p-limit": "^7.3.1","postprocessing": "^6.39.1",@@ -11156,6 +11158,12 @@"lit": "^3"}},+ "node_modules/@pinojs/redact": {+ "version": "0.4.0",+ "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz",+ "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==",+ "license": "MIT"+ },"node_modules/@pixiv/three-vrm": {"version": "3.5.3","license": "MIT",@@ -19280,6 +19288,17 @@"zod": "^3.x"}},+ "node_modules/@zone-eu/mailsplit": {+ "version": "5.4.15",+ "resolved": "https://registry.npmjs.org/@zone-eu/mailsplit/-/mailsplit-5.4.15.tgz",+ "integrity": "sha512-c7ZpxauvF4AEkDJlKDYO7iMUtMuqJMBnDWNff1cyx+d7zaBVR3iFEmXhNHOVoMmVyVF3pTZLsLIJsEFKDldOAA==",+ "license": "(MIT OR EUPL-1.1+)",+ "dependencies": {+ "libbase64": "1.3.0",+ "libmime": "5.4.2",+ "libqp": "2.1.1"+ }+ },"node_modules/abitype": {"version": "1.2.3","license": "MIT",@@ -23251,6 +23270,15 @@"iconv-lite": "^0.6.2"}},+ "node_modules/encoding-japanese": {+ "version": "2.2.0",+ "resolved": "https://registry.npmjs.org/encoding-japanese/-/encoding-japanese-2.2.0.tgz",+ "integrity": "sha512-EuJWwlHPZ1LbADuKTClvHtwbaFn4rOD+dRAbWysqEOXRc2Uui0hJInNJrsdH0c+OhJA4nrCBdSkW4DD5YxAo6A==",+ "license": "MIT",+ "engines": {+ "node": ">=8.10.0"+ }+ },"node_modules/encoding/node_modules/iconv-lite": {"version": "0.6.3","license": "MIT",@@ -25884,7 +25912,9 @@}},"node_modules/iconv-lite": {- "version": "0.7.2",+ "version": "0.7.3",+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz",+ "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==","license": "MIT","dependencies": {"safer-buffer": ">= 2.1.2 < 3.0.0"@@ -25932,6 +25962,71 @@"node": ">= 4"}},+ "node_modules/imapflow": {+ "version": "1.7.6",+ "resolved": "https://registry.npmjs.org/imapflow/-/imapflow-1.7.6.tgz",+ "integrity": "sha512-xah3HpJKHzNbO780jy+v/Gjmly9WBgPH2dl5/nNs+Q5ofjpn4bUTz0PR2kcrHGlBRsjGFOC7NTYkStakGN8Fyg==",+ "license": "MIT",+ "dependencies": {+ "@zone-eu/mailsplit": "5.4.15",+ "encoding-japanese": "2.2.0",+ "iconv-lite": "0.7.3",+ "libbase64": "1.3.0",+ "libmime": "5.4.2",+ "libqp": "2.1.1",+ "pino": "10.3.1",+ "socks": "2.8.9"+ }+ },+ "node_modules/imapflow/node_modules/pino": {+ "version": "10.3.1",+ "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz",+ "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==",+ "license": "MIT",+ "dependencies": {+ "@pinojs/redact": "^0.4.0",+ "atomic-sleep": "^1.0.0",+ "on-exit-leak-free": "^2.1.0",+ "pino-abstract-transport": "^3.0.0",+ "pino-std-serializers": "^7.0.0",+ "process-warning": "^5.0.0",+ "quick-format-unescaped": "^4.0.3",+ "real-require": "^0.2.0",+ "safe-stable-stringify": "^2.3.1",+ "sonic-boom": "^4.0.1",+ "thread-stream": "^4.0.0"+ },+ "bin": {+ "pino": "bin.js"+ }+ },+ "node_modules/imapflow/node_modules/pino-abstract-transport": {+ "version": "3.0.0",+ "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz",+ "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==",+ "license": "MIT",+ "dependencies": {+ "split2": "^4.0.0"+ }+ },+ "node_modules/imapflow/node_modules/thread-stream": {+ "version": "4.2.0",+ "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz",+ "integrity": "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==",+ "license": "MIT",+ "dependencies": {+ "real-require": "^1.0.0"+ },+ "engines": {+ "node": ">=20"+ }+ },+ "node_modules/imapflow/node_modules/thread-stream/node_modules/real-require": {+ "version": "1.0.0",+ "resolved": "https://registry.npmjs.org/real-require/-/real-require-1.0.0.tgz",+ "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==",+ "license": "MIT"+ },"node_modules/immediate": {"version": "3.0.6","license": "MIT"@@ -29304,6 +29399,24 @@"node": ">= 0.8.0"}},+ "node_modules/libbase64": {+ "version": "1.3.0",+ "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-1.3.0.tgz",+ "integrity": "sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg==",+ "license": "MIT"+ },+ "node_modules/libmime": {+ "version": "5.4.2",+ "resolved": "https://registry.npmjs.org/libmime/-/libmime-5.4.2.tgz",+ "integrity": "sha512-+IQnCOdPiufGBkOii+Ze8F7iniyBzOwvWDbn1DyExBpc9pT2B3IEMQi7GUc/PpqhNUh/sr1SG9UXDITQoR0VIA==",+ "license": "MIT",+ "dependencies": {+ "encoding-japanese": "2.2.0",+ "iconv-lite": "0.7.3",+ "libbase64": "1.3.0",+ "libqp": "2.1.1"+ }+ },"node_modules/libp2p": {"version": "2.10.0","license": "Apache-2.0 OR MIT",@@ -29346,6 +29459,12 @@"version": "1.13.5","license": "MIT"},+ "node_modules/libqp": {+ "version": "2.1.1",+ "resolved": "https://registry.npmjs.org/libqp/-/libqp-2.1.1.tgz",+ "integrity": "sha512-0Wd+GPz1O134cP62YU2GTOPNA7Qgl09XwCqM5zpBv87ERCXdfDtyKXvV7c9U22yWJh44QZqBocFnXN11K96qow==",+ "license": "MIT"+ },"node_modules/libsodium-sumo": {"version": "0.7.16","license": "ISC"@@ -31191,6 +31310,27 @@"version": "1.0.4","license": "MIT"},+ "node_modules/moment": {+ "version": "2.30.1",+ "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",+ "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==",+ "license": "MIT",+ "engines": {+ "node": "*"+ }+ },+ "node_modules/moment-timezone": {+ "version": "0.5.48",+ "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.48.tgz",+ "integrity": "sha512-f22b8LV1gbTO2ms2j2z13MuPogNoh5UzxL3nzNAYKGraILnbGc9NEE6dyiiiLv46DGRb8A4kg8UKWLjPthxBHw==",+ "license": "MIT",+ "dependencies": {+ "moment": "^2.29.4"+ },+ "engines": {+ "node": "*"+ }+ },"node_modules/mortice": {"version": "3.3.1","license": "Apache-2.0 OR MIT",@@ -31517,6 +31657,32 @@"he": "1.2.0"}},+ "node_modules/node-ical": {+ "version": "0.20.1",+ "resolved": "https://registry.npmjs.org/node-ical/-/node-ical-0.20.1.tgz",+ "integrity": "sha512-NrXgzDJd6XcyX9kDMJVA3xYCZmntY7ghA2BOdBeYr3iu8tydHOAb+68jPQhF9V2CRQ0/386X05XhmLzQUN0+Hw==",+ "license": "Apache-2.0",+ "dependencies": {+ "axios": "^1.7.7",+ "moment-timezone": "^0.5.45",+ "rrule": "2.8.1",+ "uuid": "^10.0.0"+ }+ },+ "node_modules/node-ical/node_modules/uuid": {+ "version": "10.0.0",+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",+ "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",+ "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).",+ "funding": [+ "https://github.com/sponsors/broofa",+ "https://github.com/sponsors/ctavan"+ ],+ "license": "MIT",+ "bin": {+ "uuid": "dist/bin/uuid"+ }+ },"node_modules/node-int64": {"version": "0.4.0","license": "MIT"@@ -33983,6 +34149,15 @@"@types/node": "*"}},+ "node_modules/rrule": {+ "version": "2.8.1",+ "resolved": "https://registry.npmjs.org/rrule/-/rrule-2.8.1.tgz",+ "integrity": "sha512-hM3dHSBMeaJ0Ktp7W38BJZ7O1zOgaFEsn41PDk+yHoEtfLV+PoJt9E9xAlZiWgf/iqEqionN0ebHFZIDAp+iGw==",+ "license": "BSD-3-Clause",+ "dependencies": {+ "tslib": "^2.4.0"+ }+ },"node_modules/run-parallel": {"version": "1.2.0","funding": [@@ -35475,7 +35650,6 @@"version": "4.2.0","resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz","integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",- "dev": true,"license": "MIT","engines": {"node": ">= 6.0.0",@@ -35494,7 +35668,6 @@"version": "2.8.9","resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz","integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==",- "dev": true,"license": "MIT","dependencies": {"ip-address": "^10.1.1",diff --git a/package.json b/package.jsonindex 96ce4a0ba..e5385315a 100644--- a/package.json+++ b/package.json@@ -498,6 +498,7 @@"gltf-validator": "^2.0.0-dev.3.10","helius-sdk": "^3.0.0","idb-keyval": "^6.3.0",+ "imapflow": "^1.7.6","jose": "^5.9.6","lightweight-charts": "^5.2.0","lil-gui": "^0.21.0",@@ -511,6 +512,7 @@"nanoid": "^3.3.11","nipplejs": "^1.0.4","node-html-parser": "^7.1.0",+ "node-ical": "^0.20.1","onnxruntime-web": "^1.22.0","p-limit": "^7.3.1","postprocessing": "^6.39.1",diff --git a/walk-sdk/src/companion.js b/walk-sdk/src/companion.jsindex ee72daf92..b7d2896d2 100644--- a/walk-sdk/src/companion.js+++ b/walk-sdk/src/companion.js@@ -46,6 +46,11 @@ const CORNER_RESERVE_KEY = 'walk-companion';const CURSOR_IDLE_MS = 450; // cursor still longer than this → stop walkingconst GAZE_IDLE_MS = 4000; // cursor still longer than this → gaze drifts back to the clipconst GAZE_PX_PER_M = 260; // page pixels per scene metre when aiming the head at the cursor+const BUBBLE_HOLD_MS = 5200; // default speech-bubble dwell before it retracts+// An announced message the companion was summoned for retreats this long after+// its bubble does, so the visitor sees the avatar acknowledge and walk off+// rather than vanish mid-sentence.+const ANNOUNCE_RETREAT_MS = 700;function isExcludedRoute(config) {if (typeof window === 'undefined') return true;@@ -157,7 +162,14 @@ class WalkCompanion {});}- async mount() {+ /**+ * Mount the corner companion.+ * @param {{greet?: boolean}} [opts] `greet: false` skips the page greeting+ * and the follow-up invite bubble, used when the companion is summoned to+ * deliver a specific message (see the control object's `announce`), where a+ * greeting would talk over it.+ */+ async mount({ greet = true } = {}) {if (this.mounted || isExcludedRoute(this.config)) return;if (!webglSupported()) return;this.mounted = true;@@ -175,7 +187,8 @@ class WalkCompanion {this.host?.classList.remove('is-loading');this._restoreState();this._bindEvents();- this._greetForRoute();+ if (greet) this._greetForRoute();+ else this._orientToContext();this.clock = new Timer();this._raf = requestAnimationFrame(this._tick);}@@ -504,17 +517,77 @@ class WalkCompanion {}_say(text) {- if (!this.bubble || !text) return;- this.bubble.textContent = text;- this.bubble.hidden = false;- this.bubble.classList.add('is-in');+ this.say(text);+ }++ /**+ * Show a speech bubble over the companion.+ *+ * The greeting flow calls this with no options (5.2s, plain text). Callers+ * that deliver something the visitor may want to act on pass `actions`:+ * up to two links/buttons rendered under the line, which also switches the+ * bubble to pointer-events:auto so they are clickable.+ *+ * @param {string} text+ * @param {object} [opts]+ * @param {number} [opts.hold=5200] ms before the bubble retracts; 0 keeps it+ * up until the next say()/hideBubble().+ * @param {'neutral'|'alert'} [opts.tone='neutral'] 'alert' tints the bubble.+ * @param {Array<{label:string, href?:string, onClick?:Function}>} [opts.actions]+ * @returns {boolean} false when there is no bubble to render into.+ */+ say(text, { hold = BUBBLE_HOLD_MS, tone = 'neutral', actions = null } = {}) {+ if (!this.bubble || !text) return false;+ const bubble = this.bubble;+ clearTimeout(this._bubbleTimer);+ bubble.textContent = '';++ const line = document.createElement('span');+ line.className = 'walk-companion-bubble-line';+ line.textContent = text;+ bubble.appendChild(line);++ const rendered = (actions || []).filter((a) => a && a.label).slice(0, 2);+ if (rendered.length) {+ const row = document.createElement('span');+ row.className = 'walk-companion-bubble-actions';+ for (const action of rendered) {+ // An href makes it a real link (middle-click, copy, keyboard);+ // otherwise it is a button so it never navigates by accident.+ const el = document.createElement(action.href ? 'a' : 'button');+ el.className = 'walk-companion-bubble-action';+ el.textContent = action.label;+ if (action.href) el.href = action.href;+ else el.type = 'button';+ if (typeof action.onClick === 'function') {+ el.addEventListener('click', (e) => {+ e.stopPropagation();+ action.onClick(e);+ });+ }+ row.appendChild(el);+ }+ bubble.appendChild(row);+ }++ bubble.classList.toggle('is-alert', tone === 'alert');+ bubble.classList.toggle('has-actions', rendered.length > 0);+ bubble.hidden = false;+ bubble.classList.add('is-in');+ if (hold > 0) this._bubbleTimer = setTimeout(() => this.hideBubble(), hold);+ return true;+ }++ /** Retract the speech bubble now. */+ hideBubble() {clearTimeout(this._bubbleTimer);- this._bubbleTimer = setTimeout(() => {- this.bubble.classList.remove('is-in');- setTimeout(() => {- if (this.bubble) this.bubble.hidden = true;- }, 300);- }, 5200);+ if (!this.bubble) return;+ const bubble = this.bubble;+ bubble.classList.remove('is-in');+ setTimeout(() => {+ // A newer say() may have re-opened it while this retraction ran.+ if (bubble && !bubble.classList.contains('is-in')) bubble.hidden = true;+ }, 300);}// ── Persistence ───────────────────────────────────────────────────────────@@ -640,6 +713,16 @@ function ensureStyles() {.walk-companion-close:focus-visible,.walk-companion-swap:focus-visible{outline:2px solid #7aa2ff;outline-offset:2px;opacity:1}.walk-companion-bubble{position:absolute;left:50%;bottom:calc(100% - 38px);z-index:2;transform:translateX(-50%) translateY(6px);box-sizing:border-box;max-width:min(216px,calc(100vw - 16px));width:max-content;background:rgba(18,20,28,.94);color:#f2f4f8;font:500 12.5px/1.4 system-ui,-apple-system,'Segoe UI',sans-serif;padding:8px 11px;border-radius:12px;border:1px solid rgba(255,255,255,.1);box-shadow:0 10px 28px rgba(0,0,0,.35);pointer-events:none;opacity:0;transition:opacity .3s ease,transform .3s ease;text-align:center}.walk-companion-bubble.is-in{opacity:1;transform:translateX(-50%) translateY(0)}+.walk-companion-bubble.is-alert{background:rgba(20,18,32,.96);border-color:rgba(122,162,255,.5);box-shadow:0 10px 28px rgba(0,0,0,.45),0 0 0 1px rgba(122,162,255,.18)}+.walk-companion-bubble.is-alert::after{border-top-color:rgba(20,18,32,.96)}+.walk-companion-bubble.has-actions{pointer-events:auto}+.walk-companion-bubble-line{display:block}+.walk-companion-bubble-actions{display:flex;gap:6px;justify-content:center;flex-wrap:wrap;margin-top:7px}+.walk-companion-bubble-action{appearance:none;border:1px solid rgba(255,255,255,.16);background:rgba(255,255,255,.06);color:#dbe6ff;font:600 11.5px/1 system-ui,-apple-system,'Segoe UI',sans-serif;padding:5px 9px;border-radius:999px;cursor:pointer;text-decoration:none;transition:background .15s ease,border-color .15s ease,color .15s ease}+.walk-companion-bubble-action:hover{background:rgba(122,162,255,.22);border-color:rgba(122,162,255,.55);color:#fff}+.walk-companion-bubble-action:active{background:rgba(122,162,255,.34)}+.walk-companion-bubble-action:focus-visible{outline:2px solid #7aa2ff;outline-offset:2px}+@media (pointer:coarse){.walk-companion-bubble-action{padding:9px 12px;font-size:12.5px}}.walk-companion-bubble::after{content:'';position:absolute;left:50%;top:100%;transform:translateX(-50%);border:6px solid transparent;border-top-color:rgba(18,20,28,.94)}@media (max-width:520px){.walk-companion{width:148px;height:208px;right:10px;bottom:10px}.walk-companion-bubble{font-size:11.5px;max-width:min(156px,calc(100vw - 12px))}}@media (pointer:coarse){.walk-companion-close,.walk-companion-swap{opacity:1;width:44px;height:44px;top:-7px;border:9px solid transparent;background-clip:padding-box}.walk-companion-close{right:-7px}.walk-companion-swap{right:23px}}@@ -701,6 +784,53 @@ export function createWalkCompanion(opts = {}) {instance?.openPicker();},+ /**+ * Deliver a message in person.+ *+ * If the companion is already on screen it turns to the visitor, plays a+ * gesture and says the line. If it is not (the visitor never turned it on,+ * or closed it), it is summoned silently for this one message and walks+ * off again afterwards, without flipping the persisted enabled flag, so+ * "off" stays off once the message is delivered.+ *+ * @param {string} message the line to speak. Rendered as text, never HTML.+ * @param {object} [opts]+ * @param {number} [opts.hold=7000] ms the bubble stays up.+ * @param {'neutral'|'alert'} [opts.tone='alert']+ * @param {string} [opts.emote='wave'] gesture played on arrival; falls back+ * to a wave on rigs that don't carry it.+ * @param {Array<{label:string, href?:string, onClick?:Function}>} [opts.actions]+ * @returns {Promise<boolean>} false when no avatar could be shown (route+ * excluded, no WebGL, inside an iframe) so the caller can fall back to+ * its own UI. True once the message is on screen.+ */+ async announce(message, { hold = 7000, tone = 'alert', emote = 'wave', actions } = {}) {+ if (!message) return false;+ const summoned = !instance?.mounted;+ // Transient only when the visitor has not turned the companion on: an+ // enabled-but-unmounted companion (excluded route, playground) stays+ // mounted afterwards exactly as if it had mounted normally.+ const transient = summoned && !isEnabled();+ if (summoned) {+ if (!instance) instance = new WalkCompanion(config, control);+ await instance.mount({ greet: false });+ if (!instance.mounted) return false;+ }+ const inst = instance;+ try {+ if (!inst.controller?.playEmote?.(emote)) inst.controller?.playWave?.();+ } catch {+ /* rig without emotes: the bubble still carries the message */+ }+ inst.say(message, { hold, tone, actions });+ if (!transient) return true;+ await new Promise((r) => setTimeout(r, hold + ANNOUNCE_RETREAT_MS));+ // The visitor may have turned the companion on (or swapped instances)+ // while it was delivering; never yank it away in that case.+ if (!isEnabled() && instance === inst && inst.mounted) inst.unmount();+ return true;+ },+// Re-mount the corner companion when the playground exits._wirePlaygroundReturn() {if (pgWired) return;