Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(gateway): extend chat gateways to Slack, WhatsApp, Signal, SMS and email with voice-note transcoding and a preferred channel

nirholas12ef1fc0bc08
diff --git a/api/_lib/agent-thread.js b/api/_lib/agent-thread.js
index 9b73046f6..d8122d977 100644
--- a/api/_lib/agent-thread.js
+++ b/api/_lib/agent-thread.js
@@ -1,12 +1,13 @@
// One conversation per (agent, owner), shared by every surface that talks to
-// the agent: the web copilot, the v1 messages API, Telegram and Discord. Rows
+// the agent: the web copilot, the v1 messages API and every chat gateway
+// (Telegram, Discord, Slack, WhatsApp, Signal, SMS, email). Rows
// live in agent_messages; `channel` says which surface a message came from so
// each surface can label the others ("via Telegram") while the model sees one
// continuous history.
import { sql } from './db.js';
-export const THREAD_CHANNELS = ['web', 'api', 'telegram', 'discord'];
+export const THREAD_CHANNELS = ['web', 'api', 'telegram', 'discord', 'slack', 'whatsapp', 'signal', 'sms', 'email'];
const MAX_CONTENT = 8000;
/**
diff --git a/api/_lib/gateway/audio.js b/api/_lib/gateway/audio.js
new file mode 100644
index 000000000..4601277b7
--- /dev/null
+++ b/api/_lib/gateway/audio.js
@@ -0,0 +1,128 @@
+// Audio plumbing for the chat gateways: every platform delivers voice notes in
+// its own container (Telegram, WhatsApp and Discord send Ogg/Opus, Signal and
+// iOS clients send AAC in MP4, Slack clips arrive as WebM or MP4), and every
+// platform that plays a voice note wants Ogg/Opus back. The speech lanes speak
+// WAV and listen to Ogg/Opus, FLAC or 16-bit PCM, so this module converts at
+// the edges with ffmpeg and nothing else in the gateway touches a codec.
+//
+// The ffmpeg binary comes from FFMPEG_PATH (the gateway image installs the
+// distro build there), then the ffmpeg-static package, then the PATH.
+
+import { spawn } from 'node:child_process';
+
+let resolvedPath = null;
+
+export async function ffmpegPath() {
+ if (resolvedPath) return resolvedPath;
+ if (process.env.FFMPEG_PATH) return (resolvedPath = process.env.FFMPEG_PATH);
+ try {
+ const mod = await import('ffmpeg-static');
+ if (mod?.default) return (resolvedPath = mod.default);
+ } catch {
+ // Not installed in this image: fall through to the PATH.
+ }
+ return (resolvedPath = 'ffmpeg');
+}
+
+export class AudioError extends Error {
+ constructor(code, message) {
+ super(message);
+ this.code = code;
+ }
+}
+
+/**
+ * Pipe `input` through ffmpeg with `args` (input is always stdin, output is
+ * always stdout) and resolve with the output bytes.
+ * @param {Buffer} input
+ * @param {string[]} args arguments between `-i pipe:0` and `pipe:1`
+ * @param {{ timeoutMs?:number, maxBytes?:number }} [opts]
+ * @returns {Promise<Buffer>}
+ */
+export async function runFfmpeg(input, args, { timeoutMs = 30_000, maxBytes = 32 * 1024 * 1024 } = {}) {
+ const bin = await ffmpegPath();
+ return new Promise((resolve, reject) => {
+ const proc = spawn(bin, ['-hide_banner', '-loglevel', 'error', '-i', 'pipe:0', ...args, 'pipe:1'], { stdio: ['pipe', 'pipe', 'pipe'] });
+ const out = [];
+ let size = 0;
+ let stderr = '';
+ const timer = setTimeout(() => {
+ proc.kill('SIGKILL');
+ reject(new AudioError('transcode_timeout', 'audio conversion took too long'));
+ }, timeoutMs);
+ proc.on('error', (e) => {
+ clearTimeout(timer);
+ reject(new AudioError('transcoder_unavailable', `ffmpeg could not start: ${e.message}`));
+ });
+ proc.stdout.on('data', (chunk) => {
+ size += chunk.length;
+ if (size > maxBytes) {
+ proc.kill('SIGKILL');
+ return;
+ }
+ out.push(chunk);
+ });
+ proc.stderr.on('data', (d) => { stderr = (stderr + d.toString()).slice(-600); });
+ proc.on('close', (code) => {
+ clearTimeout(timer);
+ if (size > maxBytes) return reject(new AudioError('audio_too_large', 'converted audio exceeded the size limit'));
+ if (code !== 0) return reject(new AudioError('transcode_failed', `ffmpeg exited ${code}: ${stderr.trim()}`));
+ resolve(Buffer.concat(out));
+ });
+ // A client that closes stdin early (bad container) must not crash the worker.
+ proc.stdin.on('error', () => {});
+ proc.stdin.end(input);
+ });
+}
+
+/** The speech-lane encoding a container can be sent as without conversion, or null. */
+export function nativeAsrEncoding(mimeType) {
+ const m = String(mimeType || '').toLowerCase();
+ if (/ogg|opus/.test(m)) return 'OGGOPUS';
+ if (/flac/.test(m)) return 'FLAC';
+ return null;
+}
+
+/**
+ * Turn any voice note into something the speech lane accepts: Ogg/Opus and
+ * FLAC pass through, everything else becomes 16 kHz mono 16-bit PCM.
+ * @returns {Promise<{ audio:Buffer, encoding:string, sampleRateHz:number }>}
+ */
+export async function toSpeechInput({ buffer, mimeType }) {
+ const native = nativeAsrEncoding(mimeType);
+ if (native === 'OGGOPUS') return { audio: buffer, encoding: 'OGGOPUS', sampleRateHz: 48000 };
+ if (native === 'FLAC') return { audio: buffer, encoding: 'FLAC', sampleRateHz: 16000 };
+ const pcm = await runFfmpeg(buffer, ['-vn', '-ac', '1', '-ar', '16000', '-f', 's16le', '-acodec', 'pcm_s16le']);
+ if (!pcm.length) throw new AudioError('no_audio', 'that voice note carried no audio');
+ return { audio: pcm, encoding: 'LINEAR_PCM', sampleRateHz: 16000 };
+}
+
+/** Duration in seconds of a canonical RIFF/WAVE buffer, or null when it is not one. */
+export function wavDurationSeconds(wav) {
+ if (!Buffer.isBuffer(wav) || wav.length < 44 || wav.toString('ascii', 0, 4) !== 'RIFF' || wav.toString('ascii', 8, 12) !== 'WAVE') return null;
+ let offset = 12;
+ let byteRate = null;
+ while (offset + 8 <= wav.length) {
+ const id = wav.toString('ascii', offset, offset + 4);
+ const size = wav.readUInt32LE(offset + 4);
+ if (id === 'fmt ') byteRate = wav.readUInt32LE(offset + 16);
+ if (id === 'data') {
+ if (!byteRate) return null;
+ const dataBytes = Math.min(size, wav.length - offset - 8);
+ return dataBytes / byteRate;
+ }
+ offset += 8 + size + (size % 2);
+ }
+ return null;
+}
+
+/**
+ * Encode speech for a chat voice note: Ogg/Opus, mono, 48 kHz, voice-tuned.
+ * @param {Buffer} wav
+ * @returns {Promise<{ buffer:Buffer, mimeType:string, durationSec:number|null }>}
+ */
+export async function wavToVoiceNote(wav) {
+ const buffer = await runFfmpeg(wav, ['-vn', '-ac', '1', '-ar', '48000', '-c:a', 'libopus', '-b:a', '32k', '-application', 'voip', '-f', 'ogg']);
+ if (!buffer.length) throw new AudioError('transcode_failed', 'speech conversion produced no audio');
+ return { buffer, mimeType: 'audio/ogg', durationSec: wavDurationSeconds(wav) };
+}
diff --git a/api/_lib/gateway/format.js b/api/_lib/gateway/format.js
index bc822cc57..0abcf5c0f 100644
--- a/api/_lib/gateway/format.js
+++ b/api/_lib/gateway/format.js
@@ -1,11 +1,17 @@
-// Plain-text rendering for chat replies. Both platforms get the same text:
-// Telegram is sent without a parse mode (nothing to escape, links auto-link)
-// and Discord renders it as-is, so one formatter serves both.
+// Plain-text rendering for chat replies. Every platform gets the same text:
+// Telegram is sent without a parse mode (nothing to escape, links auto-link),
+// Discord, Slack, WhatsApp and Signal render it as-is, and SMS and email carry
+// it verbatim, so one formatter serves them all.
import { env } from '../env.js';
-export const PLATFORM_LABEL = { telegram: 'Telegram', discord: 'Discord', web: 'the web', api: 'the API' };
-export const MAX_TEXT = { telegram: 4000, discord: 1900 };
+export const PLATFORM_LABEL = {
+ telegram: 'Telegram', discord: 'Discord', slack: 'Slack', whatsapp: 'WhatsApp', signal: 'Signal',
+ sms: 'SMS', email: 'email', web: 'the web', api: 'the API',
+};
+// Per-message ceilings. SMS is split by the carrier past 160 characters, so a
+// reply is kept to a few segments; email has no practical ceiling.
+export const MAX_TEXT = { telegram: 4000, discord: 1900, slack: 3500, whatsapp: 4000, signal: 4000, sms: 600, email: 20000 };
export function appOrigin() {
return env.APP_ORIGIN || 'https://three.ws';
diff --git a/api/_lib/gateway/media.js b/api/_lib/gateway/media.js
index 8aeda8c1a..69fb4b1ea 100644
--- a/api/_lib/gateway/media.js
+++ b/api/_lib/gateway/media.js
@@ -4,6 +4,7 @@
import { transcribeNvidiaAsr, nvidiaAsrConfigured } from '../asr-nvidia.js';
import { describeImage, visionConfigured } from '../vision.js';
+import { toSpeechInput, AudioError } from './audio.js';
export const MAX_VOICE_BYTES = 8 * 1024 * 1024;
export const MAX_IMAGE_BYTES = 8 * 1024 * 1024;
@@ -16,16 +17,23 @@ export class MediaError extends Error {
}
/**
- * Transcribe a chat voice note. Telegram and Discord both deliver Ogg/Opus at
- * 48 kHz, which Riva accepts directly (no transcode).
+ * Transcribe a chat voice note. Ogg/Opus (Telegram, Discord, WhatsApp) and FLAC
+ * go to Riva as-is; AAC, MP4 and WebM (Signal, iOS, Slack clips) are converted
+ * to 16 kHz PCM first (./audio.js).
* @returns {Promise<string>}
*/
export async function transcribeVoice({ buffer, mimeType = 'audio/ogg', language = 'en-US' }) {
if (!nvidiaAsrConfigured()) throw new MediaError('asr_unavailable', 'Voice notes are not available right now. Type your message instead.');
if (!buffer?.length) throw new MediaError('empty_audio', 'That voice note was empty.');
if (buffer.length > MAX_VOICE_BYTES) throw new MediaError('audio_too_large', 'That voice note is too long. Keep it under about five minutes.');
- const encoding = /ogg|opus/i.test(mimeType) ? 'OGGOPUS' : /flac/i.test(mimeType) ? 'FLAC' : 'OGGOPUS';
- const out = await transcribeNvidiaAsr({ audio: buffer, encoding, sampleRateHz: 48000, language });
+ let input;
+ try {
+ input = await toSpeechInput({ buffer, mimeType });
+ } catch (e) {
+ if (e instanceof AudioError) throw new MediaError('audio_unreadable', 'I could not read that voice note. Try recording it again, or type it.');
+ throw e;
+ }
+ const out = await transcribeNvidiaAsr({ audio: input.audio, encoding: input.encoding, sampleRateHz: input.sampleRateHz, language });
const text = String(out?.text || '').trim();
if (!text) throw new MediaError('no_speech', 'I could not make out any words in that voice note.');
return text;
diff --git a/api/_lib/gateway/platforms.js b/api/_lib/gateway/platforms.js
new file mode 100644
index 000000000..4d58f3e0b
--- /dev/null
+++ b/api/_lib/gateway/platforms.js
@@ -0,0 +1,42 @@
+// What each chat platform on the agent gateway can do. One table, read by the
+// gateway core (reply codes on channels without buttons, status lines only where
+// a message can be edited), the notification fan-out (which preference column a
+// platform delivers under) and the connections API (how the settings page
+// describes each platform). docs/chat-gateways.md renders the same facts.
+
+/**
+ * @typedef {object} PlatformInfo
+ * @property {string} label how the UI names it
+ * @property {boolean} buttons Approve / Cancel render as tappable buttons
+ * @property {boolean} canEdit a sent message can be edited in place
+ * @property {boolean} voice inbound voice notes and spoken replies
+ * @property {string|null} notifyChannel the notify-prefs column it delivers under, null when it never carries notifications
+ */
+
+/** @type {Record<string, PlatformInfo>} */
+export const PLATFORM_INFO = Object.freeze({
+ telegram: { label: 'Telegram', buttons: true, canEdit: true, voice: true, notifyChannel: 'telegram' },
+ discord: { label: 'Discord', buttons: true, canEdit: true, voice: true, notifyChannel: 'discord' },
+ slack: { label: 'Slack', buttons: true, canEdit: true, voice: true, notifyChannel: 'chat' },
+ whatsapp: { label: 'WhatsApp', buttons: true, canEdit: false, voice: true, notifyChannel: 'chat' },
+ signal: { label: 'Signal', buttons: false, canEdit: false, voice: true, notifyChannel: 'chat' },
+ sms: { label: 'SMS', buttons: false, canEdit: false, voice: false, notifyChannel: 'chat' },
+ // The owner's own inbox already receives the `email` notification channel,
+ // so an email chat never doubles it.
+ email: { label: 'Email', buttons: false, canEdit: false, voice: true, notifyChannel: null },
+});
+
+export const PLATFORMS = Object.freeze(Object.keys(PLATFORM_INFO));
+
+export function isPlatform(p) {
+ return Object.prototype.hasOwnProperty.call(PLATFORM_INFO, p);
+}
+
+export function platformInfo(p) {
+ return PLATFORM_INFO[p] || null;
+}
+
+/** Platforms that deliver notifications under a given notify-prefs column. */
+export function platformsForNotifyChannel(channel) {
+ return PLATFORMS.filter((p) => PLATFORM_INFO[p].notifyChannel === channel);
+}
diff --git a/api/_lib/gateway/store.js b/api/_lib/gateway/store.js
index bf1c5a114..46d9331d5 100644
--- a/api/_lib/gateway/store.js
+++ b/api/_lib/gateway/store.js
@@ -32,7 +32,8 @@ export async function getLinkById(id) {
export async function listLinksForUser(userId) {
return sql`
SELECT l.id, l.platform, l.platform_username, l.chat_id, l.chat_type, l.chat_title,
- l.default_agent_id, a.name AS default_agent_name, l.notify, l.created_at, l.last_seen_at
+ l.default_agent_id, a.name AS default_agent_name, l.notify, l.voice_replies, l.preferred,
+ l.created_at, l.last_seen_at
FROM gateway_links l
LEFT JOIN agent_identities a ON a.id = l.default_agent_id AND a.deleted_at IS NULL
WHERE l.user_id = ${userId} AND l.revoked_at IS NULL
@@ -41,7 +42,7 @@ export async function listLinksForUser(userId) {
export async function listNotifyLinks(userId) {
return sql`
- SELECT id, platform, chat_id FROM gateway_links
+ SELECT id, platform, chat_id, preferred, last_seen_at FROM gateway_links
WHERE user_id = ${userId} AND revoked_at IS NULL AND notify = true`;
}
@@ -50,7 +51,7 @@ export async function listNotifyLinks(userId) {
* that link unchanged; a chat paired to another account is refused, so one chat
* can never drive two accounts.
*/
-export async function createLink({ platform, platformUserId, platformUsername = null, chatId, chatType = null, chatTitle = null, userId }) {
+export async function createLink({ platform, platformUserId, platformUsername = null, chatId, chatType = null, chatTitle = null, userId, defaultAgentId = null }) {
const existing = await getLiveLink(platform, chatId);
if (existing) {
if (existing.user_id !== userId) throw new GatewayError('chat_taken', 'This chat is already paired to a different three.ws account. Send /unlink from that account first.', 409);
@@ -58,8 +59,8 @@ export async function createLink({ platform, platformUserId, platformUsername =
}
try {
const [row] = await sql`
- INSERT INTO gateway_links (platform, platform_user_id, platform_username, chat_id, chat_type, chat_title, user_id, last_seen_at)
- VALUES (${platform}, ${String(platformUserId)}, ${platformUsername}, ${String(chatId)}, ${chatType}, ${chatTitle}, ${userId}, now())
+ INSERT INTO gateway_links (platform, platform_user_id, platform_username, chat_id, chat_type, chat_title, user_id, default_agent_id, last_seen_at)
+ VALUES (${platform}, ${String(platformUserId)}, ${platformUsername}, ${String(chatId)}, ${chatType}, ${chatTitle}, ${userId}, ${defaultAgentId}, now())
RETURNING *`;
return row;
} catch (e) {
@@ -76,7 +77,7 @@ export async function createLink({ platform, platformUserId, platformUsername =
export async function revokeLink(linkId, userId) {
const [row] = await sql`
- UPDATE gateway_links SET revoked_at = now()
+ UPDATE gateway_links SET revoked_at = now(), preferred = false
WHERE id = ${linkId} AND user_id = ${userId} AND revoked_at IS NULL
RETURNING id, platform, chat_id`;
if (row) {
@@ -99,6 +100,37 @@ export async function setLinkNotify(linkId, userId, notify) {
return row || null;
}
+export async function setLinkVoiceReplies(linkId, userId, on) {
+ const [row] = await sql`
+ UPDATE gateway_links SET voice_replies = ${!!on}
+ WHERE id = ${linkId} AND user_id = ${userId} AND revoked_at IS NULL
+ RETURNING id, voice_replies`;
+ return row || null;
+}
+
+/**
+ * Make one chat the account's preferred notification channel, or clear it.
+ * The partial unique index allows one preferred live link per account, so the
+ * old one is cleared first, in the same transaction.
+ */
+export async function setLinkPreferred(linkId, userId, on) {
+ if (!on) {
+ const [row] = await sql`
+ UPDATE gateway_links SET preferred = false
+ WHERE id = ${linkId} AND user_id = ${userId} AND revoked_at IS NULL
+ RETURNING id, preferred`;
+ return row || null;
+ }
+ const [, rows] = await sql.transaction([
+ sql`UPDATE gateway_links SET preferred = false WHERE user_id = ${userId} AND preferred AND id <> ${linkId}`,
+ sql`
+ UPDATE gateway_links SET preferred = true, notify = true
+ WHERE id = ${linkId} AND user_id = ${userId} AND revoked_at IS NULL
+ RETURNING id, preferred`,
+ ]);
+ return rows?.[0] || null;
+}
+
export async function resetLinkContext(linkId) {
await sql`UPDATE gateway_links SET context_reset_at = now() WHERE id = ${linkId}`;
}