Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(gateway): extend chat gateways to Slack, WhatsApp, Signal, SMS and email with voice-note transcoding and a preferred channel
nirholas12ef1fc0bc08
diff --git a/api/_lib/agent-thread.js b/api/_lib/agent-thread.jsindex 9b73046f6..d8122d977 100644--- a/api/_lib/agent-thread.js+++ b/api/_lib/agent-thread.js@@ -1,12 +1,13 @@// One conversation per (agent, owner), shared by every surface that talks to-// the agent: the web copilot, the v1 messages API, Telegram and Discord. Rows+// the agent: the web copilot, the v1 messages API and every chat gateway+// (Telegram, Discord, Slack, WhatsApp, Signal, SMS, email). Rows// live in agent_messages; `channel` says which surface a message came from so// each surface can label the others ("via Telegram") while the model sees one// continuous history.import { sql } from './db.js';-export const THREAD_CHANNELS = ['web', 'api', 'telegram', 'discord'];+export const THREAD_CHANNELS = ['web', 'api', 'telegram', 'discord', 'slack', 'whatsapp', 'signal', 'sms', 'email'];const MAX_CONTENT = 8000;/**diff --git a/api/_lib/gateway/audio.js b/api/_lib/gateway/audio.jsnew file mode 100644index 000000000..4601277b7--- /dev/null+++ b/api/_lib/gateway/audio.js@@ -0,0 +1,128 @@+// Audio plumbing for the chat gateways: every platform delivers voice notes in+// its own container (Telegram, WhatsApp and Discord send Ogg/Opus, Signal and+// iOS clients send AAC in MP4, Slack clips arrive as WebM or MP4), and every+// platform that plays a voice note wants Ogg/Opus back. The speech lanes speak+// WAV and listen to Ogg/Opus, FLAC or 16-bit PCM, so this module converts at+// the edges with ffmpeg and nothing else in the gateway touches a codec.+//+// The ffmpeg binary comes from FFMPEG_PATH (the gateway image installs the+// distro build there), then the ffmpeg-static package, then the PATH.++import { spawn } from 'node:child_process';++let resolvedPath = null;++export async function ffmpegPath() {+ if (resolvedPath) return resolvedPath;+ if (process.env.FFMPEG_PATH) return (resolvedPath = process.env.FFMPEG_PATH);+ try {+ const mod = await import('ffmpeg-static');+ if (mod?.default) return (resolvedPath = mod.default);+ } catch {+ // Not installed in this image: fall through to the PATH.+ }+ return (resolvedPath = 'ffmpeg');+}++export class AudioError extends Error {+ constructor(code, message) {+ super(message);+ this.code = code;+ }+}++/**+ * Pipe `input` through ffmpeg with `args` (input is always stdin, output is+ * always stdout) and resolve with the output bytes.+ * @param {Buffer} input+ * @param {string[]} args arguments between `-i pipe:0` and `pipe:1`+ * @param {{ timeoutMs?:number, maxBytes?:number }} [opts]+ * @returns {Promise<Buffer>}+ */+export async function runFfmpeg(input, args, { timeoutMs = 30_000, maxBytes = 32 * 1024 * 1024 } = {}) {+ const bin = await ffmpegPath();+ return new Promise((resolve, reject) => {+ const proc = spawn(bin, ['-hide_banner', '-loglevel', 'error', '-i', 'pipe:0', ...args, 'pipe:1'], { stdio: ['pipe', 'pipe', 'pipe'] });+ const out = [];+ let size = 0;+ let stderr = '';+ const timer = setTimeout(() => {+ proc.kill('SIGKILL');+ reject(new AudioError('transcode_timeout', 'audio conversion took too long'));+ }, timeoutMs);+ proc.on('error', (e) => {+ clearTimeout(timer);+ reject(new AudioError('transcoder_unavailable', `ffmpeg could not start: ${e.message}`));+ });+ proc.stdout.on('data', (chunk) => {+ size += chunk.length;+ if (size > maxBytes) {+ proc.kill('SIGKILL');+ return;+ }+ out.push(chunk);+ });+ proc.stderr.on('data', (d) => { stderr = (stderr + d.toString()).slice(-600); });+ proc.on('close', (code) => {+ clearTimeout(timer);+ if (size > maxBytes) return reject(new AudioError('audio_too_large', 'converted audio exceeded the size limit'));+ if (code !== 0) return reject(new AudioError('transcode_failed', `ffmpeg exited ${code}: ${stderr.trim()}`));+ resolve(Buffer.concat(out));+ });+ // A client that closes stdin early (bad container) must not crash the worker.+ proc.stdin.on('error', () => {});+ proc.stdin.end(input);+ });+}++/** The speech-lane encoding a container can be sent as without conversion, or null. */+export function nativeAsrEncoding(mimeType) {+ const m = String(mimeType || '').toLowerCase();+ if (/ogg|opus/.test(m)) return 'OGGOPUS';+ if (/flac/.test(m)) return 'FLAC';+ return null;+}++/**+ * Turn any voice note into something the speech lane accepts: Ogg/Opus and+ * FLAC pass through, everything else becomes 16 kHz mono 16-bit PCM.+ * @returns {Promise<{ audio:Buffer, encoding:string, sampleRateHz:number }>}+ */+export async function toSpeechInput({ buffer, mimeType }) {+ const native = nativeAsrEncoding(mimeType);+ if (native === 'OGGOPUS') return { audio: buffer, encoding: 'OGGOPUS', sampleRateHz: 48000 };+ if (native === 'FLAC') return { audio: buffer, encoding: 'FLAC', sampleRateHz: 16000 };+ const pcm = await runFfmpeg(buffer, ['-vn', '-ac', '1', '-ar', '16000', '-f', 's16le', '-acodec', 'pcm_s16le']);+ if (!pcm.length) throw new AudioError('no_audio', 'that voice note carried no audio');+ return { audio: pcm, encoding: 'LINEAR_PCM', sampleRateHz: 16000 };+}++/** Duration in seconds of a canonical RIFF/WAVE buffer, or null when it is not one. */+export function wavDurationSeconds(wav) {+ if (!Buffer.isBuffer(wav) || wav.length < 44 || wav.toString('ascii', 0, 4) !== 'RIFF' || wav.toString('ascii', 8, 12) !== 'WAVE') return null;+ let offset = 12;+ let byteRate = null;+ while (offset + 8 <= wav.length) {+ const id = wav.toString('ascii', offset, offset + 4);+ const size = wav.readUInt32LE(offset + 4);+ if (id === 'fmt ') byteRate = wav.readUInt32LE(offset + 16);+ if (id === 'data') {+ if (!byteRate) return null;+ const dataBytes = Math.min(size, wav.length - offset - 8);+ return dataBytes / byteRate;+ }+ offset += 8 + size + (size % 2);+ }+ return null;+}++/**+ * Encode speech for a chat voice note: Ogg/Opus, mono, 48 kHz, voice-tuned.+ * @param {Buffer} wav+ * @returns {Promise<{ buffer:Buffer, mimeType:string, durationSec:number|null }>}+ */+export async function wavToVoiceNote(wav) {+ const buffer = await runFfmpeg(wav, ['-vn', '-ac', '1', '-ar', '48000', '-c:a', 'libopus', '-b:a', '32k', '-application', 'voip', '-f', 'ogg']);+ if (!buffer.length) throw new AudioError('transcode_failed', 'speech conversion produced no audio');+ return { buffer, mimeType: 'audio/ogg', durationSec: wavDurationSeconds(wav) };+}diff --git a/api/_lib/gateway/format.js b/api/_lib/gateway/format.jsindex bc822cc57..0abcf5c0f 100644--- a/api/_lib/gateway/format.js+++ b/api/_lib/gateway/format.js@@ -1,11 +1,17 @@-// Plain-text rendering for chat replies. Both platforms get the same text:-// Telegram is sent without a parse mode (nothing to escape, links auto-link)-// and Discord renders it as-is, so one formatter serves both.+// Plain-text rendering for chat replies. Every platform gets the same text:+// Telegram is sent without a parse mode (nothing to escape, links auto-link),+// Discord, Slack, WhatsApp and Signal render it as-is, and SMS and email carry+// it verbatim, so one formatter serves them all.import { env } from '../env.js';-export const PLATFORM_LABEL = { telegram: 'Telegram', discord: 'Discord', web: 'the web', api: 'the API' };-export const MAX_TEXT = { telegram: 4000, discord: 1900 };+export const PLATFORM_LABEL = {+ telegram: 'Telegram', discord: 'Discord', slack: 'Slack', whatsapp: 'WhatsApp', signal: 'Signal',+ sms: 'SMS', email: 'email', web: 'the web', api: 'the API',+};+// Per-message ceilings. SMS is split by the carrier past 160 characters, so a+// reply is kept to a few segments; email has no practical ceiling.+export const MAX_TEXT = { telegram: 4000, discord: 1900, slack: 3500, whatsapp: 4000, signal: 4000, sms: 600, email: 20000 };export function appOrigin() {return env.APP_ORIGIN || 'https://three.ws';diff --git a/api/_lib/gateway/media.js b/api/_lib/gateway/media.jsindex 8aeda8c1a..69fb4b1ea 100644--- a/api/_lib/gateway/media.js+++ b/api/_lib/gateway/media.js@@ -4,6 +4,7 @@import { transcribeNvidiaAsr, nvidiaAsrConfigured } from '../asr-nvidia.js';import { describeImage, visionConfigured } from '../vision.js';+import { toSpeechInput, AudioError } from './audio.js';export const MAX_VOICE_BYTES = 8 * 1024 * 1024;export const MAX_IMAGE_BYTES = 8 * 1024 * 1024;@@ -16,16 +17,23 @@ export class MediaError extends Error {}/**- * Transcribe a chat voice note. Telegram and Discord both deliver Ogg/Opus at- * 48 kHz, which Riva accepts directly (no transcode).+ * Transcribe a chat voice note. Ogg/Opus (Telegram, Discord, WhatsApp) and FLAC+ * go to Riva as-is; AAC, MP4 and WebM (Signal, iOS, Slack clips) are converted+ * to 16 kHz PCM first (./audio.js).* @returns {Promise<string>}*/export async function transcribeVoice({ buffer, mimeType = 'audio/ogg', language = 'en-US' }) {if (!nvidiaAsrConfigured()) throw new MediaError('asr_unavailable', 'Voice notes are not available right now. Type your message instead.');if (!buffer?.length) throw new MediaError('empty_audio', 'That voice note was empty.');if (buffer.length > MAX_VOICE_BYTES) throw new MediaError('audio_too_large', 'That voice note is too long. Keep it under about five minutes.');- const encoding = /ogg|opus/i.test(mimeType) ? 'OGGOPUS' : /flac/i.test(mimeType) ? 'FLAC' : 'OGGOPUS';- const out = await transcribeNvidiaAsr({ audio: buffer, encoding, sampleRateHz: 48000, language });+ let input;+ try {+ input = await toSpeechInput({ buffer, mimeType });+ } catch (e) {+ if (e instanceof AudioError) throw new MediaError('audio_unreadable', 'I could not read that voice note. Try recording it again, or type it.');+ throw e;+ }+ const out = await transcribeNvidiaAsr({ audio: input.audio, encoding: input.encoding, sampleRateHz: input.sampleRateHz, language });const text = String(out?.text || '').trim();if (!text) throw new MediaError('no_speech', 'I could not make out any words in that voice note.');return text;diff --git a/api/_lib/gateway/platforms.js b/api/_lib/gateway/platforms.jsnew file mode 100644index 000000000..4d58f3e0b--- /dev/null+++ b/api/_lib/gateway/platforms.js@@ -0,0 +1,42 @@+// What each chat platform on the agent gateway can do. One table, read by the+// gateway core (reply codes on channels without buttons, status lines only where+// a message can be edited), the notification fan-out (which preference column a+// platform delivers under) and the connections API (how the settings page+// describes each platform). docs/chat-gateways.md renders the same facts.++/**+ * @typedef {object} PlatformInfo+ * @property {string} label how the UI names it+ * @property {boolean} buttons Approve / Cancel render as tappable buttons+ * @property {boolean} canEdit a sent message can be edited in place+ * @property {boolean} voice inbound voice notes and spoken replies+ * @property {string|null} notifyChannel the notify-prefs column it delivers under, null when it never carries notifications+ */++/** @type {Record<string, PlatformInfo>} */+export const PLATFORM_INFO = Object.freeze({+ telegram: { label: 'Telegram', buttons: true, canEdit: true, voice: true, notifyChannel: 'telegram' },+ discord: { label: 'Discord', buttons: true, canEdit: true, voice: true, notifyChannel: 'discord' },+ slack: { label: 'Slack', buttons: true, canEdit: true, voice: true, notifyChannel: 'chat' },+ whatsapp: { label: 'WhatsApp', buttons: true, canEdit: false, voice: true, notifyChannel: 'chat' },+ signal: { label: 'Signal', buttons: false, canEdit: false, voice: true, notifyChannel: 'chat' },+ sms: { label: 'SMS', buttons: false, canEdit: false, voice: false, notifyChannel: 'chat' },+ // The owner's own inbox already receives the `email` notification channel,+ // so an email chat never doubles it.+ email: { label: 'Email', buttons: false, canEdit: false, voice: true, notifyChannel: null },+});++export const PLATFORMS = Object.freeze(Object.keys(PLATFORM_INFO));++export function isPlatform(p) {+ return Object.prototype.hasOwnProperty.call(PLATFORM_INFO, p);+}++export function platformInfo(p) {+ return PLATFORM_INFO[p] || null;+}++/** Platforms that deliver notifications under a given notify-prefs column. */+export function platformsForNotifyChannel(channel) {+ return PLATFORMS.filter((p) => PLATFORM_INFO[p].notifyChannel === channel);+}diff --git a/api/_lib/gateway/store.js b/api/_lib/gateway/store.jsindex bf1c5a114..46d9331d5 100644--- a/api/_lib/gateway/store.js+++ b/api/_lib/gateway/store.js@@ -32,7 +32,8 @@ export async function getLinkById(id) {export async function listLinksForUser(userId) {return sql`SELECT l.id, l.platform, l.platform_username, l.chat_id, l.chat_type, l.chat_title,- l.default_agent_id, a.name AS default_agent_name, l.notify, l.created_at, l.last_seen_at+ l.default_agent_id, a.name AS default_agent_name, l.notify, l.voice_replies, l.preferred,+ l.created_at, l.last_seen_atFROM gateway_links lLEFT JOIN agent_identities a ON a.id = l.default_agent_id AND a.deleted_at IS NULLWHERE l.user_id = ${userId} AND l.revoked_at IS NULL@@ -41,7 +42,7 @@ export async function listLinksForUser(userId) {export async function listNotifyLinks(userId) {return sql`- SELECT id, platform, chat_id FROM gateway_links+ SELECT id, platform, chat_id, preferred, last_seen_at FROM gateway_linksWHERE user_id = ${userId} AND revoked_at IS NULL AND notify = true`;}@@ -50,7 +51,7 @@ export async function listNotifyLinks(userId) {* that link unchanged; a chat paired to another account is refused, so one chat* can never drive two accounts.*/-export async function createLink({ platform, platformUserId, platformUsername = null, chatId, chatType = null, chatTitle = null, userId }) {+export async function createLink({ platform, platformUserId, platformUsername = null, chatId, chatType = null, chatTitle = null, userId, defaultAgentId = null }) {const existing = await getLiveLink(platform, chatId);if (existing) {if (existing.user_id !== userId) throw new GatewayError('chat_taken', 'This chat is already paired to a different three.ws account. Send /unlink from that account first.', 409);@@ -58,8 +59,8 @@ export async function createLink({ platform, platformUserId, platformUsername =}try {const [row] = await sql`- INSERT INTO gateway_links (platform, platform_user_id, platform_username, chat_id, chat_type, chat_title, user_id, last_seen_at)- VALUES (${platform}, ${String(platformUserId)}, ${platformUsername}, ${String(chatId)}, ${chatType}, ${chatTitle}, ${userId}, now())+ INSERT INTO gateway_links (platform, platform_user_id, platform_username, chat_id, chat_type, chat_title, user_id, default_agent_id, last_seen_at)+ VALUES (${platform}, ${String(platformUserId)}, ${platformUsername}, ${String(chatId)}, ${chatType}, ${chatTitle}, ${userId}, ${defaultAgentId}, now())RETURNING *`;return row;} catch (e) {@@ -76,7 +77,7 @@ export async function createLink({ platform, platformUserId, platformUsername =export async function revokeLink(linkId, userId) {const [row] = await sql`- UPDATE gateway_links SET revoked_at = now()+ UPDATE gateway_links SET revoked_at = now(), preferred = falseWHERE id = ${linkId} AND user_id = ${userId} AND revoked_at IS NULLRETURNING id, platform, chat_id`;if (row) {@@ -99,6 +100,37 @@ export async function setLinkNotify(linkId, userId, notify) {return row || null;}+export async function setLinkVoiceReplies(linkId, userId, on) {+ const [row] = await sql`+ UPDATE gateway_links SET voice_replies = ${!!on}+ WHERE id = ${linkId} AND user_id = ${userId} AND revoked_at IS NULL+ RETURNING id, voice_replies`;+ return row || null;+}++/**+ * Make one chat the account's preferred notification channel, or clear it.+ * The partial unique index allows one preferred live link per account, so the+ * old one is cleared first, in the same transaction.+ */+export async function setLinkPreferred(linkId, userId, on) {+ if (!on) {+ const [row] = await sql`+ UPDATE gateway_links SET preferred = false+ WHERE id = ${linkId} AND user_id = ${userId} AND revoked_at IS NULL+ RETURNING id, preferred`;+ return row || null;+ }+ const [, rows] = await sql.transaction([+ sql`UPDATE gateway_links SET preferred = false WHERE user_id = ${userId} AND preferred AND id <> ${linkId}`,+ sql`+ UPDATE gateway_links SET preferred = true, notify = true+ WHERE id = ${linkId} AND user_id = ${userId} AND revoked_at IS NULL+ RETURNING id, preferred`,+ ]);+ return rows?.[0] || null;+}+export async function resetLinkContext(linkId) {await sql`UPDATE gateway_links SET context_reset_at = now() WHERE id = ${linkId}`;}