Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(companion): let the corner avatar walk on and deliver a user's own important messages

nirholasd1f7b4595b8d
diff --git a/api/_lib/companion/crypto.js b/api/_lib/companion/crypto.js
new file mode 100644
index 000000000..d1332b2e5
--- /dev/null
+++ b/api/_lib/companion/crypto.js
@@ -0,0 +1,71 @@
+// Encrypt/decrypt a companion source's BYOK credentials at rest.
+//
+// Same AES-256-GCM + HKDF construction as api/_lib/provider-keys.js, with its
+// own salt so a bot token can never be decrypted by a key derived for another
+// key space. The plaintext is a small JSON blob (bot token, IMAP password, a
+// calendar's secret ICS URL), so it round-trips as a string.
+
+import { webcrypto } from 'node:crypto';
+import { env } from '../env.js';
+
+const subtle = globalThis.crypto?.subtle || webcrypto.subtle;
+
+function randomBytes(n) {
+ const b = new Uint8Array(n);
+ (globalThis.crypto || webcrypto).getRandomValues(b);
+ return b;
+}
+
+async function deriveKey() {
+ const raw = new TextEncoder().encode(env.JWT_SECRET);
+ const base = await subtle.importKey('raw', raw, 'HKDF', false, ['deriveKey']);
+ return subtle.deriveKey(
+ {
+ name: 'HKDF',
+ hash: 'SHA-256',
+ salt: new TextEncoder().encode('companion-source-config-v1'),
+ info: new Uint8Array(0),
+ },
+ base,
+ { name: 'AES-GCM', length: 256 },
+ false,
+ ['encrypt', 'decrypt'],
+ );
+}
+
+export async function encryptConfig(config) {
+ const key = await deriveKey();
+ const iv = randomBytes(12);
+ const data = new TextEncoder().encode(JSON.stringify(config));
+ const ct = await subtle.encrypt({ name: 'AES-GCM', iv }, key, data);
+ const buf = new Uint8Array(iv.length + ct.byteLength);
+ buf.set(iv, 0);
+ buf.set(new Uint8Array(ct), iv.length);
+ return Buffer.from(buf).toString('base64');
+}
+
+export async function decryptConfig(ciphertext) {
+ const key = await deriveKey();
+ const raw = Buffer.from(ciphertext, 'base64');
+ const plain = await subtle.decrypt({ name: 'AES-GCM', iv: raw.subarray(0, 12) }, key, raw.subarray(12));
+ return JSON.parse(new TextDecoder().decode(plain));
+}
+
+// What the UI is allowed to see about a stored credential: enough to recognise
+// which account is connected, never enough to reuse it elsewhere.
+export function redactConfig(kind, config) {
+ if (!config || typeof config !== 'object') return {};
+ if (kind === 'telegram') {
+ const token = String(config.bot_token || '');
+ return { bot_username: config.bot_username || null, bot_token_hint: token ? `${token.slice(0, 6)}…${token.slice(-4)}` : null };
+ }
+ if (kind === 'calendar') {
+ let host = null;
+ try { host = new URL(config.ics_url).host; } catch { host = null; }
+ return { ics_host: host, lookahead_minutes: config.lookahead_minutes || 60 };
+ }
+ if (kind === 'email') {
+ return { host: config.host || null, port: config.port || null, user: config.user || null, folder: config.folder || 'INBOX' };
+ }
+ return {};
+}
diff --git a/api/_lib/migrations/20260828140000_companion_harness.sql b/api/_lib/migrations/20260828140000_companion_harness.sql
new file mode 100644
index 000000000..3b9b27da7
--- /dev/null
+++ b/api/_lib/migrations/20260828140000_companion_harness.sql
@@ -0,0 +1,108 @@
+begin;
+
+-- The Companion: a personal 3D assistant that delivers a user's own important
+-- messages in person. Everything here is per-user and BYOK: the credentials for
+-- each source live encrypted in companion_sources.config_encrypted (AES-256-GCM,
+-- api/_lib/companion/crypto.js), never in plaintext columns.
+
+create table if not exists companion_settings (
+ user_id uuid primary key references users(id) on delete cascade,
+ enabled boolean not null default true,
+ -- Only events scoring at or above this get a spoken delivery (0-100).
+ threshold integer not null default 60,
+ -- Quiet hours in the user's own timezone; a delivery inside the window is
+ -- stored and shown in the feed but never spoken or pushed.
+ quiet_start smallint,
+ quiet_end smallint,
+ timezone text not null default 'UTC',
+ -- Default stage presence when a sender has no contact-specific avatar.
+ avatar_glb_url text,
+ voice text not null default 'alloy',
+ -- Bearer for the phone/desktop bridge (POST /api/companion/ingest).
+ ingest_token text not null,
+ push_enabled boolean not null default true,
+ created_at timestamptz not null default now(),
+ updated_at timestamptz not null default now(),
+ constraint companion_settings_threshold_chk check (threshold between 0 and 100),
+ constraint companion_settings_quiet_chk
+ check ((quiet_start is null) = (quiet_end is null)
+ and (quiet_start is null or (quiet_start between 0 and 23 and quiet_end between 0 and 23)))
+);
+
+create unique index if not exists companion_settings_ingest_token_uniq
+ on companion_settings (ingest_token);
+
+create table if not exists companion_sources (
+ id uuid primary key default gen_random_uuid(),
+ user_id uuid not null references users(id) on delete cascade,
+ kind text not null,
+ label text not null,
+ config_encrypted text not null,
+ -- Lane cursor: telegram update offset, imap uid, calendar seen-uid set.
+ cursor jsonb not null default '{}'::jsonb,
+ enabled boolean not null default true,
+ status text not null default 'pending',
+ last_error text,
+ last_polled_at timestamptz,
+ last_event_at timestamptz,
+ created_at timestamptz not null default now(),
+ constraint companion_sources_kind_chk check (kind in ('telegram', 'calendar', 'email')),
+ constraint companion_sources_status_chk check (status in ('pending', 'ok', 'error'))
+);
+
+create index if not exists companion_sources_user_idx on companion_sources (user_id, kind);
+create index if not exists companion_sources_poll_idx on companion_sources (enabled, last_polled_at);
+
+-- Who the message is from, and who shows up to deliver it. One row per identity
+-- the user cares about (a telegram @handle, an email address, a phone number).
+create table if not exists companion_contacts (
+ id uuid primary key default gen_random_uuid(),
+ user_id uuid not null references users(id) on delete cascade,
+ identifier text not null,
+ display_name text not null,
+ avatar_glb_url text,
+ avatar_image_url text,
+ voice text,
+ -- Added to every event from this identity, so family outranks a newsletter.
+ priority_boost integer not null default 0,
+ created_at timestamptz not null default now(),
+ constraint companion_contacts_boost_chk check (priority_boost between -100 and 100)
+);
+
+create unique index if not exists companion_contacts_identity_uniq
+ on companion_contacts (user_id, lower(identifier));
+
+create table if not exists companion_events (
+ id uuid primary key default gen_random_uuid(),
+ user_id uuid not null references users(id) on delete cascade,
+ source_id uuid references companion_sources(id) on delete set null,
+ source_kind text not null,
+ -- Stable per-lane id (telegram message id, imap uid, calendar uid+start,
+ -- bridge-supplied id) so a re-poll never speaks the same message twice.
+ external_id text not null,
+ contact_id uuid references companion_contacts(id) on delete set null,
+ sender text,
+ sender_id text,
+ title text not null,
+ body text,
+ url text,
+ importance integer not null default 0,
+ reason text,
+ spoken_line text,
+ triage_engine text not null default 'rules',
+ occurs_at timestamptz,
+ delivered_at timestamptz,
+ dismissed_at timestamptz,
+ created_at timestamptz not null default now(),
+ constraint companion_events_importance_chk check (importance between 0 and 100)
+);
+
+create unique index if not exists companion_events_dedupe_uniq
+ on companion_events (user_id, source_kind, external_id);
+create index if not exists companion_events_feed_idx
+ on companion_events (user_id, created_at desc);
+create index if not exists companion_events_pending_idx
+ on companion_events (user_id, delivered_at, importance desc)
+ where delivered_at is null and dismissed_at is null;
+
+commit;
diff --git a/api/_lib/notify-prefs.js b/api/_lib/notify-prefs.js
index 128da1795..07ea0e309 100644
--- a/api/_lib/notify-prefs.js
+++ b/api/_lib/notify-prefs.js
@@ -4,7 +4,8 @@
// A notification has a `type` (e.g. 'skill_purchased'). Types roll up into a
// small set of user-facing CATEGORIES. The user controls, per category, which
// CHANNELS deliver it. Channels: in_app (the bell inbox), push (Web Push),
-// email (Resend transactional), telegram (per-account bot alerts).
+// email (Resend transactional), telegram (per-account bot alerts), avatar (the
+// corner companion walks on and says it out loud, see src/notification-herald.js).
//
// Preferences are stored sparsely in notification_preferences.prefs as
// { categories: { sales: { push: false }, ... }, telegram_chat_id: '123' }
@@ -59,7 +60,7 @@ export const CATEGORIES = [
},
];
-export const CHANNELS = ['in_app', 'push', 'email', 'telegram'];
+export const CHANNELS = ['in_app', 'push', 'email', 'telegram', 'avatar'];
/**
* Channels that always deliver for a category, regardless of preferences.
@@ -124,23 +125,39 @@ export function categoryForType(type) {
return TYPE_CATEGORY[type] || 'account';
}
+/**
+ * The whole type to category map, for clients that have to resolve a category
+ * without a round trip per notification (the avatar channel runs in the
+ * browser). Served by GET /api/notifications/preferences so the browser never
+ * keeps its own copy of this table.
+ */
+export function typeCategoryMap() {
+ return { ...TYPE_CATEGORY };
+}
+
// Default channel matrix. in_app is on everywhere (the bell is the baseline).
// push is on by default *for users who have subscribed a device* — a user with
// no push subscription simply never receives one, so defaulting it on is not
// spammy. email defaults on only for money + security; off for the higher-
// frequency social/irl/alerts categories. telegram is always opt-in (needs a
// linked chat id) so it defaults off.
+// avatar is the most interruptive channel there is: the corner companion walks
+// on screen and says the line out loud. It defaults on only for the three
+// categories worth stopping the visitor for (money in, a creation that
+// finished while they waited, account and security), and off for the
+// higher-frequency ones. Every category stays togglable in the preference
+// center either way.
const DEFAULTS = {
- sales: { in_app: true, push: true, email: true, telegram: false },
- purchases: { in_app: true, push: true, email: true, telegram: false },
- social: { in_app: true, push: true, email: false, telegram: false },
- irl: { in_app: true, push: true, email: false, telegram: false },
- alerts: { in_app: true, push: true, email: false, telegram: true },
+ sales: { in_app: true, push: true, email: true, telegram: false, avatar: true },
+ purchases: { in_app: true, push: true, email: true, telegram: false, avatar: false },
+ social: { in_app: true, push: true, email: false, telegram: false, avatar: false },
+ irl: { in_app: true, push: true, email: false, telegram: false, avatar: false },
+ alerts: { in_app: true, push: true, email: false, telegram: true, avatar: false },
// Only unattended completions notify (api/cron/forge-finalize.js), so email
// defaulting on is the feature, not spam: the user left the page and asked
// to hear back.
- creations: { in_app: true, push: true, email: true, telegram: false },
- account: { in_app: true, push: true, email: true, telegram: false },
+ creations: { in_app: true, push: true, email: true, telegram: false, avatar: true },
+ account: { in_app: true, push: true, email: true, telegram: false, avatar: true },
};
/** The full default matrix, used to seed the preference-center UI. */
diff --git a/api/notifications/preferences.js b/api/notifications/preferences.js
index 8b15e3658..1ecb5ab6e 100644
--- a/api/notifications/preferences.js
+++ b/api/notifications/preferences.js
@@ -1,7 +1,8 @@
// Unified notification preference center.
//
// GET /api/notifications/preferences
-// → { categories: [...], channels: [...], prefs: {...}, push: {...} }
+// → { categories: [...], channels: [...], type_categories: {...},
+// prefs: {...}, push: {...} }
// the full resolved matrix + metadata the UI renders from.
// PUT /api/notifications/preferences { categories: {...}, telegram_chat_id }
// → merge a sanitised sparse override onto what is already stored; unknown
@@ -25,6 +26,7 @@ import {
readStoredPrefs,
resolvePrefs,
sanitizePrefs,
+ typeCategoryMap,
} from '../_lib/notify-prefs.js';
const putBody = z.object({
@@ -55,6 +57,10 @@ export default wrap(async (req, res) => {
return json(res, 200, {
categories: CATEGORIES,
channels: CHANNELS,
+ // The avatar channel is delivered client-side (src/notification-herald.js),
+ // so the browser needs the same type to category mapping the server
+ // gates push and email with. Shipping it here keeps one source of truth.
+ type_categories: typeCategoryMap(),
prefs,
push: { subscribed_devices: pushRow?.count ?? 0 },
});
diff --git a/package-lock.json b/package-lock.json
index 71c48b0f7..f158287ca 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -168,6 +168,7 @@
"gltf-validator": "^2.0.0-dev.3.10",
"helius-sdk": "^3.0.0",
"idb-keyval": "^6.3.0",
+ "imapflow": "^1.7.6",
"jose": "^5.9.6",
"lightweight-charts": "^5.2.0",
"lil-gui": "^0.21.0",
@@ -181,6 +182,7 @@
"nanoid": "^3.3.11",
"nipplejs": "^1.0.4",
"node-html-parser": "^7.1.0",
+ "node-ical": "^0.20.1",
"onnxruntime-web": "^1.22.0",
"p-limit": "^7.3.1",
"postprocessing": "^6.39.1",
@@ -11156,6 +11158,12 @@
"lit": "^3"
}
},
+ "node_modules/@pinojs/redact": {
+ "version": "0.4.0",
+ "resolved": "https://registry.npmjs.org/@pinojs/redact/-/redact-0.4.0.tgz",
+ "integrity": "sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==",
+ "license": "MIT"
+ },
"node_modules/@pixiv/three-vrm": {
"version": "3.5.3",
"license": "MIT",
@@ -19280,6 +19288,17 @@
"zod": "^3.x"
}
},
+ "node_modules/@zone-eu/mailsplit": {
+ "version": "5.4.15",
+ "resolved": "https://registry.npmjs.org/@zone-eu/mailsplit/-/mailsplit-5.4.15.tgz",
+ "integrity": "sha512-c7ZpxauvF4AEkDJlKDYO7iMUtMuqJMBnDWNff1cyx+d7zaBVR3iFEmXhNHOVoMmVyVF3pTZLsLIJsEFKDldOAA==",
+ "license": "(MIT OR EUPL-1.1+)",
+ "dependencies": {
+ "libbase64": "1.3.0",
+ "libmime": "5.4.2",
+ "libqp": "2.1.1"
+ }
+ },
"node_modules/abitype": {
"version": "1.2.3",
"license": "MIT",
@@ -23251,6 +23270,15 @@
"iconv-lite": "^0.6.2"
}
},
+ "node_modules/encoding-japanese": {
+ "version": "2.2.0",
+ "resolved": "https://registry.npmjs.org/encoding-japanese/-/encoding-japanese-2.2.0.tgz",
+ "integrity": "sha512-EuJWwlHPZ1LbADuKTClvHtwbaFn4rOD+dRAbWysqEOXRc2Uui0hJInNJrsdH0c+OhJA4nrCBdSkW4DD5YxAo6A==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=8.10.0"
+ }
+ },
"node_modules/encoding/node_modules/iconv-lite": {
"version": "0.6.3",
"license": "MIT",
@@ -25884,7 +25912,9 @@
}
},
"node_modules/iconv-lite": {
- "version": "0.7.2",
+ "version": "0.7.3",
+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.3.tgz",
+ "integrity": "sha512-IKXpvIzjnC9XTAUbVBcMfGS0EPaIXtW6v+zr+RRp+hqULEpo0owZax6wyRwPOJbWbzjYspQwusTsfVr0ifh4uQ==",
"license": "MIT",
"dependencies": {
"safer-buffer": ">= 2.1.2 < 3.0.0"
@@ -25932,6 +25962,71 @@
"node": ">= 4"
}
},
+ "node_modules/imapflow": {
+ "version": "1.7.6",
+ "resolved": "https://registry.npmjs.org/imapflow/-/imapflow-1.7.6.tgz",
+ "integrity": "sha512-xah3HpJKHzNbO780jy+v/Gjmly9WBgPH2dl5/nNs+Q5ofjpn4bUTz0PR2kcrHGlBRsjGFOC7NTYkStakGN8Fyg==",
+ "license": "MIT",
+ "dependencies": {
+ "@zone-eu/mailsplit": "5.4.15",
+ "encoding-japanese": "2.2.0",
+ "iconv-lite": "0.7.3",
+ "libbase64": "1.3.0",
+ "libmime": "5.4.2",
+ "libqp": "2.1.1",
+ "pino": "10.3.1",
+ "socks": "2.8.9"
+ }
+ },
+ "node_modules/imapflow/node_modules/pino": {
+ "version": "10.3.1",
+ "resolved": "https://registry.npmjs.org/pino/-/pino-10.3.1.tgz",
+ "integrity": "sha512-r34yH/GlQpKZbU1BvFFqOjhISRo1MNx1tWYsYvmj6KIRHSPMT2+yHOEb1SG6NMvRoHRF0a07kCOox/9yakl1vg==",
+ "license": "MIT",
+ "dependencies": {
+ "@pinojs/redact": "^0.4.0",
+ "atomic-sleep": "^1.0.0",
+ "on-exit-leak-free": "^2.1.0",
+ "pino-abstract-transport": "^3.0.0",
+ "pino-std-serializers": "^7.0.0",
+ "process-warning": "^5.0.0",
+ "quick-format-unescaped": "^4.0.3",
+ "real-require": "^0.2.0",
+ "safe-stable-stringify": "^2.3.1",
+ "sonic-boom": "^4.0.1",
+ "thread-stream": "^4.0.0"
+ },
+ "bin": {
+ "pino": "bin.js"
+ }
+ },
+ "node_modules/imapflow/node_modules/pino-abstract-transport": {
+ "version": "3.0.0",
+ "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-3.0.0.tgz",
+ "integrity": "sha512-wlfUczU+n7Hy/Ha5j9a/gZNy7We5+cXp8YL+X+PG8S0KXxw7n/JXA3c46Y0zQznIJ83URJiwy7Lh56WLokNuxg==",
+ "license": "MIT",
+ "dependencies": {
+ "split2": "^4.0.0"
+ }
+ },
+ "node_modules/imapflow/node_modules/thread-stream": {
+ "version": "4.2.0",
+ "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-4.2.0.tgz",
+ "integrity": "sha512-e2zZ96wSChazBsbENf/Pcm/4swHt2cEKQ92rhUjkL9GCKiTDJIaTBenjE/m9DXi0QBmTMDkFDdOomUy20A1tDQ==",
+ "license": "MIT",
+ "dependencies": {
+ "real-require": "^1.0.0"
+ },
+ "engines": {
+ "node": ">=20"
+ }
+ },
+ "node_modules/imapflow/node_modules/thread-stream/node_modules/real-require": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/real-require/-/real-require-1.0.0.tgz",
+ "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==",
+ "license": "MIT"
+ },
"node_modules/immediate": {
"version": "3.0.6",
"license": "MIT"
@@ -29304,6 +29399,24 @@
"node": ">= 0.8.0"
}
},
+ "node_modules/libbase64": {
+ "version": "1.3.0",
+ "resolved": "https://registry.npmjs.org/libbase64/-/libbase64-1.3.0.tgz",
+ "integrity": "sha512-GgOXd0Eo6phYgh0DJtjQ2tO8dc0IVINtZJeARPeiIJqge+HdsWSuaDTe8ztQ7j/cONByDZ3zeB325AHiv5O0dg==",
+ "license": "MIT"
+ },
+ "node_modules/libmime": {
+ "version": "5.4.2",
+ "resolved": "https://registry.npmjs.org/libmime/-/libmime-5.4.2.tgz",
+ "integrity": "sha512-+IQnCOdPiufGBkOii+Ze8F7iniyBzOwvWDbn1DyExBpc9pT2B3IEMQi7GUc/PpqhNUh/sr1SG9UXDITQoR0VIA==",
+ "license": "MIT",
+ "dependencies": {
+ "encoding-japanese": "2.2.0",
+ "iconv-lite": "0.7.3",
+ "libbase64": "1.3.0",
+ "libqp": "2.1.1"
+ }
+ },
"node_modules/libp2p": {
"version": "2.10.0",
"license": "Apache-2.0 OR MIT",
@@ -29346,6 +29459,12 @@
"version": "1.13.5",
"license": "MIT"
},
+ "node_modules/libqp": {
+ "version": "2.1.1",
+ "resolved": "https://registry.npmjs.org/libqp/-/libqp-2.1.1.tgz",
+ "integrity": "sha512-0Wd+GPz1O134cP62YU2GTOPNA7Qgl09XwCqM5zpBv87ERCXdfDtyKXvV7c9U22yWJh44QZqBocFnXN11K96qow==",
+ "license": "MIT"
+ },
"node_modules/libsodium-sumo": {
"version": "0.7.16",
"license": "ISC"
@@ -31191,6 +31310,27 @@
"version": "1.0.4",
"license": "MIT"
},
+ "node_modules/moment": {
+ "version": "2.30.1",
+ "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",
+ "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==",
+ "license": "MIT",
+ "engines": {
+ "node": "*"
+ }
+ },
+ "node_modules/moment-timezone": {
+ "version": "0.5.48",
+ "resolved": "https://registry.npmjs.org/moment-timezone/-/moment-timezone-0.5.48.tgz",
+ "integrity": "sha512-f22b8LV1gbTO2ms2j2z13MuPogNoh5UzxL3nzNAYKGraILnbGc9NEE6dyiiiLv46DGRb8A4kg8UKWLjPthxBHw==",
+ "license": "MIT",
+ "dependencies": {
+ "moment": "^2.29.4"
+ },
+ "engines": {
+ "node": "*"
+ }
+ },
"node_modules/mortice": {
"version": "3.3.1",
"license": "Apache-2.0 OR MIT",
@@ -31517,6 +31657,32 @@
"he": "1.2.0"
}
},
+ "node_modules/node-ical": {
+ "version": "0.20.1",
+ "resolved": "https://registry.npmjs.org/node-ical/-/node-ical-0.20.1.tgz",
+ "integrity": "sha512-NrXgzDJd6XcyX9kDMJVA3xYCZmntY7ghA2BOdBeYr3iu8tydHOAb+68jPQhF9V2CRQ0/386X05XhmLzQUN0+Hw==",
+ "license": "Apache-2.0",
+ "dependencies": {
+ "axios": "^1.7.7",
+ "moment-timezone": "^0.5.45",
+ "rrule": "2.8.1",
+ "uuid": "^10.0.0"
+ }
+ },
+ "node_modules/node-ical/node_modules/uuid": {
+ "version": "10.0.0",
+ "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz",
+ "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==",
+ "deprecated": "uuid@10 and below is no longer supported. For ESM codebases, update to uuid@latest. For CommonJS codebases, use uuid@11 (but be aware this version will likely be deprecated in 2028).",
+ "funding": [
+ "https://github.com/sponsors/broofa",
+ "https://github.com/sponsors/ctavan"
+ ],
+ "license": "MIT",
+ "bin": {
+ "uuid": "dist/bin/uuid"
+ }
+ },
"node_modules/node-int64": {
"version": "0.4.0",
"license": "MIT"
@@ -33983,6 +34149,15 @@
"@types/node": "*"
}
},
+ "node_modules/rrule": {
+ "version": "2.8.1",
+ "resolved": "https://registry.npmjs.org/rrule/-/rrule-2.8.1.tgz",
+ "integrity": "sha512-hM3dHSBMeaJ0Ktp7W38BJZ7O1zOgaFEsn41PDk+yHoEtfLV+PoJt9E9xAlZiWgf/iqEqionN0ebHFZIDAp+iGw==",
+ "license": "BSD-3-Clause",
+ "dependencies": {
+ "tslib": "^2.4.0"
+ }
+ },
"node_modules/run-parallel": {
"version": "1.2.0",
"funding": [
@@ -35475,7 +35650,6 @@
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz",
"integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==",
- "dev": true,
"license": "MIT",
"engines": {
"node": ">= 6.0.0",
@@ -35494,7 +35668,6 @@
"version": "2.8.9",
"resolved": "https://registry.npmjs.org/socks/-/socks-2.8.9.tgz",
"integrity": "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw==",
- "dev": true,
"license": "MIT",
"dependencies": {
"ip-address": "^10.1.1",
diff --git a/package.json b/package.json
index 96ce4a0ba..e5385315a 100644
--- a/package.json
+++ b/package.json
@@ -498,6 +498,7 @@
"gltf-validator": "^2.0.0-dev.3.10",
"helius-sdk": "^3.0.0",
"idb-keyval": "^6.3.0",
+ "imapflow": "^1.7.6",
"jose": "^5.9.6",
"lightweight-charts": "^5.2.0",
"lil-gui": "^0.21.0",
@@ -511,6 +512,7 @@
"nanoid": "^3.3.11",
"nipplejs": "^1.0.4",
"node-html-parser": "^7.1.0",
+ "node-ical": "^0.20.1",
"onnxruntime-web": "^1.22.0",
"p-limit": "^7.3.1",
"postprocessing": "^6.39.1",
diff --git a/walk-sdk/src/companion.js b/walk-sdk/src/companion.js
index ee72daf92..b7d2896d2 100644
--- a/walk-sdk/src/companion.js
+++ b/walk-sdk/src/companion.js
@@ -46,6 +46,11 @@ const CORNER_RESERVE_KEY = 'walk-companion';
const CURSOR_IDLE_MS = 450; // cursor still longer than this → stop walking
const GAZE_IDLE_MS = 4000; // cursor still longer than this → gaze drifts back to the clip
const GAZE_PX_PER_M = 260; // page pixels per scene metre when aiming the head at the cursor
+const BUBBLE_HOLD_MS = 5200; // default speech-bubble dwell before it retracts
+// An announced message the companion was summoned for retreats this long after
+// its bubble does, so the visitor sees the avatar acknowledge and walk off
+// rather than vanish mid-sentence.
+const ANNOUNCE_RETREAT_MS = 700;
function isExcludedRoute(config) {
if (typeof window === 'undefined') return true;
@@ -157,7 +162,14 @@ class WalkCompanion {
});
}
- async mount() {
+ /**
+ * Mount the corner companion.
+ * @param {{greet?: boolean}} [opts] `greet: false` skips the page greeting
+ * and the follow-up invite bubble, used when the companion is summoned to
+ * deliver a specific message (see the control object's `announce`), where a
+ * greeting would talk over it.
+ */
+ async mount({ greet = true } = {}) {
if (this.mounted || isExcludedRoute(this.config)) return;
if (!webglSupported()) return;
this.mounted = true;
@@ -175,7 +187,8 @@ class WalkCompanion {
this.host?.classList.remove('is-loading');
this._restoreState();
this._bindEvents();
- this._greetForRoute();
+ if (greet) this._greetForRoute();
+ else this._orientToContext();
this.clock = new Timer();
this._raf = requestAnimationFrame(this._tick);
}
@@ -504,17 +517,77 @@ class WalkCompanion {
}
_say(text) {
- if (!this.bubble || !text) return;
- this.bubble.textContent = text;
- this.bubble.hidden = false;
- this.bubble.classList.add('is-in');
+ this.say(text);
+ }
+
+ /**
+ * Show a speech bubble over the companion.
+ *
+ * The greeting flow calls this with no options (5.2s, plain text). Callers
+ * that deliver something the visitor may want to act on pass `actions`:
+ * up to two links/buttons rendered under the line, which also switches the
+ * bubble to pointer-events:auto so they are clickable.
+ *
+ * @param {string} text
+ * @param {object} [opts]
+ * @param {number} [opts.hold=5200] ms before the bubble retracts; 0 keeps it
+ * up until the next say()/hideBubble().
+ * @param {'neutral'|'alert'} [opts.tone='neutral'] 'alert' tints the bubble.
+ * @param {Array<{label:string, href?:string, onClick?:Function}>} [opts.actions]
+ * @returns {boolean} false when there is no bubble to render into.
+ */
+ say(text, { hold = BUBBLE_HOLD_MS, tone = 'neutral', actions = null } = {}) {
+ if (!this.bubble || !text) return false;
+ const bubble = this.bubble;
+ clearTimeout(this._bubbleTimer);
+ bubble.textContent = '';
+
+ const line = document.createElement('span');
+ line.className = 'walk-companion-bubble-line';
+ line.textContent = text;
+ bubble.appendChild(line);
+
+ const rendered = (actions || []).filter((a) => a && a.label).slice(0, 2);
+ if (rendered.length) {
+ const row = document.createElement('span');
+ row.className = 'walk-companion-bubble-actions';
+ for (const action of rendered) {
+ // An href makes it a real link (middle-click, copy, keyboard);
+ // otherwise it is a button so it never navigates by accident.
+ const el = document.createElement(action.href ? 'a' : 'button');
+ el.className = 'walk-companion-bubble-action';
+ el.textContent = action.label;
+ if (action.href) el.href = action.href;
+ else el.type = 'button';
+ if (typeof action.onClick === 'function') {
+ el.addEventListener('click', (e) => {
+ e.stopPropagation();
+ action.onClick(e);
+ });
+ }
+ row.appendChild(el);
+ }
+ bubble.appendChild(row);
+ }
+
+ bubble.classList.toggle('is-alert', tone === 'alert');
+ bubble.classList.toggle('has-actions', rendered.length > 0);
+ bubble.hidden = false;
+ bubble.classList.add('is-in');
+ if (hold > 0) this._bubbleTimer = setTimeout(() => this.hideBubble(), hold);
+ return true;
+ }
+
+ /** Retract the speech bubble now. */
+ hideBubble() {
clearTimeout(this._bubbleTimer);
- this._bubbleTimer = setTimeout(() => {
- this.bubble.classList.remove('is-in');
- setTimeout(() => {
- if (this.bubble) this.bubble.hidden = true;
- }, 300);
- }, 5200);
+ if (!this.bubble) return;
+ const bubble = this.bubble;
+ bubble.classList.remove('is-in');
+ setTimeout(() => {
+ // A newer say() may have re-opened it while this retraction ran.
+ if (bubble && !bubble.classList.contains('is-in')) bubble.hidden = true;
+ }, 300);
}
// ── Persistence ───────────────────────────────────────────────────────────
@@ -640,6 +713,16 @@ function ensureStyles() {
.walk-companion-close:focus-visible,.walk-companion-swap:focus-visible{outline:2px solid #7aa2ff;outline-offset:2px;opacity:1}
.walk-companion-bubble{position:absolute;left:50%;bottom:calc(100% - 38px);z-index:2;transform:translateX(-50%) translateY(6px);box-sizing:border-box;max-width:min(216px,calc(100vw - 16px));width:max-content;background:rgba(18,20,28,.94);color:#f2f4f8;font:500 12.5px/1.4 system-ui,-apple-system,'Segoe UI',sans-serif;padding:8px 11px;border-radius:12px;border:1px solid rgba(255,255,255,.1);box-shadow:0 10px 28px rgba(0,0,0,.35);pointer-events:none;opacity:0;transition:opacity .3s ease,transform .3s ease;text-align:center}
.walk-companion-bubble.is-in{opacity:1;transform:translateX(-50%) translateY(0)}
+.walk-companion-bubble.is-alert{background:rgba(20,18,32,.96);border-color:rgba(122,162,255,.5);box-shadow:0 10px 28px rgba(0,0,0,.45),0 0 0 1px rgba(122,162,255,.18)}
+.walk-companion-bubble.is-alert::after{border-top-color:rgba(20,18,32,.96)}
+.walk-companion-bubble.has-actions{pointer-events:auto}
+.walk-companion-bubble-line{display:block}
+.walk-companion-bubble-actions{display:flex;gap:6px;justify-content:center;flex-wrap:wrap;margin-top:7px}
+.walk-companion-bubble-action{appearance:none;border:1px solid rgba(255,255,255,.16);background:rgba(255,255,255,.06);color:#dbe6ff;font:600 11.5px/1 system-ui,-apple-system,'Segoe UI',sans-serif;padding:5px 9px;border-radius:999px;cursor:pointer;text-decoration:none;transition:background .15s ease,border-color .15s ease,color .15s ease}
+.walk-companion-bubble-action:hover{background:rgba(122,162,255,.22);border-color:rgba(122,162,255,.55);color:#fff}
+.walk-companion-bubble-action:active{background:rgba(122,162,255,.34)}
+.walk-companion-bubble-action:focus-visible{outline:2px solid #7aa2ff;outline-offset:2px}
+@media (pointer:coarse){.walk-companion-bubble-action{padding:9px 12px;font-size:12.5px}}
.walk-companion-bubble::after{content:'';position:absolute;left:50%;top:100%;transform:translateX(-50%);border:6px solid transparent;border-top-color:rgba(18,20,28,.94)}
@media (max-width:520px){.walk-companion{width:148px;height:208px;right:10px;bottom:10px}.walk-companion-bubble{font-size:11.5px;max-width:min(156px,calc(100vw - 12px))}}
@media (pointer:coarse){.walk-companion-close,.walk-companion-swap{opacity:1;width:44px;height:44px;top:-7px;border:9px solid transparent;background-clip:padding-box}.walk-companion-close{right:-7px}.walk-companion-swap{right:23px}}
@@ -701,6 +784,53 @@ export function createWalkCompanion(opts = {}) {
instance?.openPicker();
},
+ /**
+ * Deliver a message in person.
+ *
+ * If the companion is already on screen it turns to the visitor, plays a
+ * gesture and says the line. If it is not (the visitor never turned it on,
+ * or closed it), it is summoned silently for this one message and walks
+ * off again afterwards, without flipping the persisted enabled flag, so
+ * "off" stays off once the message is delivered.
+ *
+ * @param {string} message the line to speak. Rendered as text, never HTML.
+ * @param {object} [opts]
+ * @param {number} [opts.hold=7000] ms the bubble stays up.
+ * @param {'neutral'|'alert'} [opts.tone='alert']
+ * @param {string} [opts.emote='wave'] gesture played on arrival; falls back
+ * to a wave on rigs that don't carry it.
+ * @param {Array<{label:string, href?:string, onClick?:Function}>} [opts.actions]
+ * @returns {Promise<boolean>} false when no avatar could be shown (route
+ * excluded, no WebGL, inside an iframe) so the caller can fall back to
+ * its own UI. True once the message is on screen.
+ */
+ async announce(message, { hold = 7000, tone = 'alert', emote = 'wave', actions } = {}) {
+ if (!message) return false;
+ const summoned = !instance?.mounted;
+ // Transient only when the visitor has not turned the companion on: an
+ // enabled-but-unmounted companion (excluded route, playground) stays
+ // mounted afterwards exactly as if it had mounted normally.
+ const transient = summoned && !isEnabled();
+ if (summoned) {
+ if (!instance) instance = new WalkCompanion(config, control);
+ await instance.mount({ greet: false });
+ if (!instance.mounted) return false;
+ }
+ const inst = instance;
+ try {
+ if (!inst.controller?.playEmote?.(emote)) inst.controller?.playWave?.();
+ } catch {
+ /* rig without emotes: the bubble still carries the message */
+ }
+ inst.say(message, { hold, tone, actions });
+ if (!transient) return true;
+ await new Promise((r) => setTimeout(r, hold + ANNOUNCE_RETREAT_MS));
+ // The visitor may have turned the companion on (or swapped instances)
+ // while it was delivering; never yank it away in that case.
+ if (!isEnabled() && instance === inst && inst.mounted) inst.unmount();
+ return true;
+ },
+
// Re-mount the corner companion when the playground exits.
_wirePlaygroundReturn() {
if (pgWired) return;