Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(gateway): spoken replies, reply-code friendly approvals and Slack, WhatsApp and SMS webhook verification

nirholas66d378154407
diff --git a/api/_lib/gateway/approvals.js b/api/_lib/gateway/approvals.js
index c124bf6dd..828f374b2 100644
--- a/api/_lib/gateway/approvals.js
+++ b/api/_lib/gateway/approvals.js
@@ -126,11 +126,21 @@ export async function handleAction({ gw, event, link }) {
}
const ref = preview.message_ref || event.messageRef || null;
const base = previewText(preview.proposal);
- const settle = async (line) => { if (ref) await gw.editMessage(ref, `${base}\n\n${line}`, { choices: [] }).catch(() => {}); };
+ // A channel that can edit rewrites the preview in place and drops its
+ // buttons. One that cannot (SMS, Signal, email: approved by reply code)
+ // gets the outcome as a new message, and skips the transient acks that
+ // would otherwise arrive as a second, redundant text.
+ const editable = gw.canEdit !== false;
+ const settle = async (line) => {
+ if (!editable) return gw.sendText(event.chatId, line).catch(() => {});
+ if (ref) await gw.editMessage(ref, `${base}\n\n${line}`, { choices: [] }).catch(() => {});
+ };
+ const ackThenSettle = (text) => (editable ? gw.ackAction(event, text) : null);
if (verb === 'cancel') {
const done = await cancelPreview(preview.id);
- await gw.ackAction(event, done ? 'Cancelled' : `Already ${preview.status}`);
+ if (done) await ackThenSettle('Cancelled');
+ else await gw.ackAction(event, `Already ${preview.status}`);
if (done) {
await settle('Cancelled. Nothing was sent.');
await appendThreadMessage({ agentId: preview.agent_id, userId: preview.user_id, role: 'assistant', content: `Cancelled the ${preview.kind} preview. Nothing was sent.`, channel: event.platform }).catch(() => {});
@@ -141,7 +151,7 @@ export async function handleAction({ gw, event, link }) {
if (preview.status !== 'pending') return gw.ackAction(event, `Already ${preview.status}.`);
if (new Date(preview.expires_at) <= new Date()) {
await expirePreview(preview.id);
- await gw.ackAction(event, 'This preview expired.');
+ await ackThenSettle('This preview expired.');
return settle('Expired. Ask your agent again for a fresh quote.');
}
const rl = await limits.gatewayMessage(link.id);
@@ -149,7 +159,7 @@ export async function handleAction({ gw, event, link }) {
const claimed = await claimPreview(preview.id);
if (!claimed) return gw.ackAction(event, 'This preview was already handled.');
- await gw.ackAction(event, 'Approved. Executing...');
+ await ackThenSettle('Approved. Executing...');
await settle('Approved. Executing...');
let result;
diff --git a/api/_lib/gateway/audio.js b/api/_lib/gateway/audio.js
index 4601277b7..d23568ce5 100644
--- a/api/_lib/gateway/audio.js
+++ b/api/_lib/gateway/audio.js
@@ -1,128 +1,197 @@
-// Audio plumbing for the chat gateways: every platform delivers voice notes in
-// its own container (Telegram, WhatsApp and Discord send Ogg/Opus, Signal and
-// iOS clients send AAC in MP4, Slack clips arrive as WebM or MP4), and every
-// platform that plays a voice note wants Ogg/Opus back. The speech lanes speak
-// WAV and listen to Ogg/Opus, FLAC or 16-bit PCM, so this module converts at
-// the edges with ffmpeg and nothing else in the gateway touches a codec.
+// Audio for the chat gateways: turning whatever a platform delivers into
+// something the speech lane can hear, and turning an agent's reply into a voice
+// note every platform can play.
//
-// The ffmpeg binary comes from FFMPEG_PATH (the gateway image installs the
-// distro build there), then the ffmpeg-static package, then the PATH.
+// Inbound: Telegram, Discord and WhatsApp send Ogg/Opus, which the NVIDIA Riva
+// lane (the same one behind POST /api/asr) accepts as is. Slack clips arrive as
+// WebM or MP4, Signal as AAC and MMS as AMR, so those are transcoded to Ogg/Opus
+// with ffmpeg first. When Riva is unreachable or unconfigured, the clip goes to
+// Gemini on Vertex AI (platform GCP credits) instead, so a voice note still
+// lands as text.
+//
+// Outbound: a reply is synthesized on the platform's server-side TTS lanes
+// (api/_lib/voice-providers.js synthesizeVoice), starting with the agent's own
+// configured voice when it lives on one of those lanes, then NVIDIA Magpie,
+// Gemini on Vertex, the free Edge lane and OpenAI. The clip is always delivered
+// as a 48 kHz mono Ogg/Opus voice note: the one container Telegram shows as a
+// voice bubble, WhatsApp as a voice message, and Signal, Slack and Discord play.
import { spawn } from 'node:child_process';
+import { createRequire } from 'node:module';
+import { sql } from '../db.js';
+import { env } from '../env.js';
+import { transcribeNvidiaAsr, nvidiaAsrConfigured } from '../asr-nvidia.js';
+import { synthesizeVoice } from '../voice-providers.js';
+import { geminiTtsConfigured } from '../tts-gemini.js';
+import { nvidiaTtsConfigured } from '../tts-nvidia.js';
+import { getGcpAccessToken } from '../gcp-auth.js';
-let resolvedPath = null;
+export const SPOKEN_MAX_CHARS = 700;
+const FFMPEG_TIMEOUT_MS = 30_000;
+const SERVER_LANES = ['nvidia', 'gemini', 'edge', 'openai'];
+const OPUS_ARGS = ['-vn', '-ac', '1', '-ar', '48000', '-c:a', 'libopus', '-b:a', '32k', '-application', 'voip', '-f', 'ogg'];
-export async function ffmpegPath() {
- if (resolvedPath) return resolvedPath;
- if (process.env.FFMPEG_PATH) return (resolvedPath = process.env.FFMPEG_PATH);
+/**
+ * The ffmpeg binary: FFMPEG_PATH when set (the gateway image installs the
+ * distro ffmpeg), the ffmpeg-static binary when its install script ran, else
+ * `ffmpeg` on PATH.
+ */
+export function ffmpegPath() {
+ if (process.env.FFMPEG_PATH) return process.env.FFMPEG_PATH;
try {
- const mod = await import('ffmpeg-static');
- if (mod?.default) return (resolvedPath = mod.default);
+ const p = createRequire(import.meta.url)('ffmpeg-static');
+ if (typeof p === 'string' && p) return p;
} catch {
- // Not installed in this image: fall through to the PATH.
- }
- return (resolvedPath = 'ffmpeg');
-}
-
-export class AudioError extends Error {
- constructor(code, message) {
- super(message);
- this.code = code;
+ // ffmpeg-static is optional here: the PATH binary below serves instead.
}
+ return 'ffmpeg';
}
-/**
- * Pipe `input` through ffmpeg with `args` (input is always stdin, output is
- * always stdout) and resolve with the output bytes.
- * @param {Buffer} input
- * @param {string[]} args arguments between `-i pipe:0` and `pipe:1`
- * @param {{ timeoutMs?:number, maxBytes?:number }} [opts]
- * @returns {Promise<Buffer>}
- */
-export async function runFfmpeg(input, args, { timeoutMs = 30_000, maxBytes = 32 * 1024 * 1024 } = {}) {
- const bin = await ffmpegPath();
+/** Run ffmpeg over a buffer: stdin in, stdout out. */
+export function runFfmpeg(input, args, { timeoutMs = FFMPEG_TIMEOUT_MS } = {}) {
return new Promise((resolve, reject) => {
- const proc = spawn(bin, ['-hide_banner', '-loglevel', 'error', '-i', 'pipe:0', ...args, 'pipe:1'], { stdio: ['pipe', 'pipe', 'pipe'] });
+ const proc = spawn(ffmpegPath(), ['-hide_banner', '-loglevel', 'error', '-i', 'pipe:0', ...args, 'pipe:1']);
const out = [];
- let size = 0;
- let stderr = '';
- const timer = setTimeout(() => {
- proc.kill('SIGKILL');
- reject(new AudioError('transcode_timeout', 'audio conversion took too long'));
- }, timeoutMs);
- proc.on('error', (e) => {
- clearTimeout(timer);
- reject(new AudioError('transcoder_unavailable', `ffmpeg could not start: ${e.message}`));
- });
- proc.stdout.on('data', (chunk) => {
- size += chunk.length;
- if (size > maxBytes) {
- proc.kill('SIGKILL');
- return;
- }
- out.push(chunk);
- });
- proc.stderr.on('data', (d) => { stderr = (stderr + d.toString()).slice(-600); });
+ let err = '';
+ const timer = setTimeout(() => { proc.kill('SIGKILL'); reject(Object.assign(new Error('ffmpeg timed out'), { code: 'transcode_timeout' })); }, timeoutMs);
+ proc.stdout.on('data', (d) => out.push(d));
+ proc.stderr.on('data', (d) => { err += d; });
+ proc.on('error', (e) => { clearTimeout(timer); reject(Object.assign(e, { code: 'transcode_unavailable' })); });
proc.on('close', (code) => {
clearTimeout(timer);
- if (size > maxBytes) return reject(new AudioError('audio_too_large', 'converted audio exceeded the size limit'));
- if (code !== 0) return reject(new AudioError('transcode_failed', `ffmpeg exited ${code}: ${stderr.trim()}`));
- resolve(Buffer.concat(out));
+ if (code === 0 && out.length) resolve(Buffer.concat(out));
+ else reject(Object.assign(new Error(`ffmpeg exited ${code}: ${err.trim().slice(0, 300)}`), { code: 'transcode_failed' }));
});
- // A client that closes stdin early (bad container) must not crash the worker.
proc.stdin.on('error', () => {});
proc.stdin.end(input);
});
}
-/** The speech-lane encoding a container can be sent as without conversion, or null. */
-export function nativeAsrEncoding(mimeType) {
- const m = String(mimeType || '').toLowerCase();
- if (/ogg|opus/.test(m)) return 'OGGOPUS';
- if (/flac/.test(m)) return 'FLAC';
- return null;
+/** True when a clip is already Ogg/Opus and Riva can take it untouched. */
+export function isOggOpus(mimeType) {
+ return /^audio\/(ogg|opus)/i.test(String(mimeType || ''));
}
-/**
- * Turn any voice note into something the speech lane accepts: Ogg/Opus and
- * FLAC pass through, everything else becomes 16 kHz mono 16-bit PCM.
- * @returns {Promise<{ audio:Buffer, encoding:string, sampleRateHz:number }>}
- */
-export async function toSpeechInput({ buffer, mimeType }) {
- const native = nativeAsrEncoding(mimeType);
- if (native === 'OGGOPUS') return { audio: buffer, encoding: 'OGGOPUS', sampleRateHz: 48000 };
- if (native === 'FLAC') return { audio: buffer, encoding: 'FLAC', sampleRateHz: 16000 };
- const pcm = await runFfmpeg(buffer, ['-vn', '-ac', '1', '-ar', '16000', '-f', 's16le', '-acodec', 'pcm_s16le']);
- if (!pcm.length) throw new AudioError('no_audio', 'that voice note carried no audio');
- return { audio: pcm, encoding: 'LINEAR_PCM', sampleRateHz: 16000 };
+/** Any recorded audio as mono 48 kHz Ogg/Opus. Ogg input passes through untouched. */
+export async function toOggOpus(buffer, mimeType = null) {
+ if (isOggOpus(mimeType)) return buffer;
+ return runFfmpeg(buffer, OPUS_ARGS);
+}
+
+async function transcribeWithGemini(ogg) {
+ const project = process.env.GOOGLE_CLOUD_PROJECT;
+ if (!project) throw Object.assign(new Error('Vertex AI is not configured'), { code: 'not_configured' });
+ const location = process.env.GOOGLE_CLOUD_LOCATION_GEMINI || 'global';
+ const host = location === 'global' ? 'aiplatform.googleapis.com' : `${location}-aiplatform.googleapis.com`;
+ const model = (process.env.VERTEX_GEMINI_MODEL || 'google/gemini-2.5-flash').replace(/^google\//, '');
+ const token = await getGcpAccessToken();
+ const r = await fetch(`https://${host}/v1/projects/${project}/locations/${location}/publishers/google/models/${model}:generateContent`, {
+ method: 'POST',
+ headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
+ body: JSON.stringify({
+ contents: [{
+ role: 'user',
+ parts: [
+ { inlineData: { mimeType: 'audio/ogg', data: ogg.toString('base64') } },
+ { text: 'Transcribe this voice note verbatim. Reply with the transcript only, no commentary. If there is no speech, reply with nothing.' },
+ ],
+ }],
+ generationConfig: { temperature: 0, maxOutputTokens: 1024, thinkingConfig: { thinkingBudget: 0 } },
+ }),
+ signal: AbortSignal.timeout(30_000),
+ });
+ const j = await r.json().catch(() => null);
+ if (!r.ok) throw Object.assign(new Error(j?.error?.message || `vertex ${r.status}`), { code: 'provider_error' });
+ return (j?.candidates?.[0]?.content?.parts || []).map((p) => p.text || '').join('').trim();
}
-/** Duration in seconds of a canonical RIFF/WAVE buffer, or null when it is not one. */
-export function wavDurationSeconds(wav) {
- if (!Buffer.isBuffer(wav) || wav.length < 44 || wav.toString('ascii', 0, 4) !== 'RIFF' || wav.toString('ascii', 8, 12) !== 'WAVE') return null;
- let offset = 12;
- let byteRate = null;
- while (offset + 8 <= wav.length) {
- const id = wav.toString('ascii', offset, offset + 4);
- const size = wav.readUInt32LE(offset + 4);
- if (id === 'fmt ') byteRate = wav.readUInt32LE(offset + 16);
- if (id === 'data') {
- if (!byteRate) return null;
- const dataBytes = Math.min(size, wav.length - offset - 8);
- return dataBytes / byteRate;
+/** Whether any speech-to-text lane can serve. */
+export function speechConfigured() {
+ return nvidiaAsrConfigured() || Boolean(process.env.GOOGLE_CLOUD_PROJECT);
+}
+
+/**
+ * Transcribe a voice note in any container. Riva first; Gemini on Vertex when
+ * Riva is down or unconfigured.
+ * @returns {Promise<string>} the transcript (empty when nothing was said)
+ */
+export async function transcribeAudio({ buffer, mimeType, language = 'en-US' }) {
+ const ogg = await toOggOpus(buffer, mimeType);
+ if (nvidiaAsrConfigured()) {
+ try {
+ const out = await transcribeNvidiaAsr({ audio: ogg, encoding: 'OGGOPUS', sampleRateHz: 48000, language });
+ return String(out?.text || '').trim();
+ } catch (e) {
+ if (!process.env.GOOGLE_CLOUD_PROJECT) throw e;
+ console.warn('[gateway] riva failed, trying gemini', e?.code || e?.message);
}
- offset += 8 + size + (size % 2);
}
- return null;
+ return transcribeWithGemini(ogg);
+}
+
+/**
+ * The part of a reply worth saying out loud: markdown, links and long base58
+ * strings (addresses, signatures) removed, cut at a sentence boundary so the
+ * clip never stops mid-word.
+ */
+export function spokenText(reply, max = SPOKEN_MAX_CHARS) {
+ let s = String(reply || '')
+ .replace(/```[\s\S]*?```/g, ' ')
+ .replace(/\[([^\]]+)\]\((?:https?:[^)]+)\)/g, '$1')
+ .replace(/https?:\/\/\S+/g, '')
+ .replace(/\b[1-9A-HJ-NP-Za-km-z]{32,88}\b/g, '')
+ .replace(/[*_`#>]/g, '')
+ .replace(/\n{2,}/g, '. ')
+ .replace(/\s+/g, ' ')
+ .replace(/\s+([.,!?])/g, '$1')
+ .trim();
+ if (s.length <= max) return s;
+ s = s.slice(0, max);
+ const cut = Math.max(s.lastIndexOf('. '), s.lastIndexOf('! '), s.lastIndexOf('? '));
+ return (cut > max * 0.4 ? s.slice(0, cut + 1) : `${s.slice(0, s.lastIndexOf(' '))}.`).trim();
+}
+
+function laneConfigured(lane) {
+ if (lane === 'nvidia') return nvidiaTtsConfigured();
+ if (lane === 'gemini') return geminiTtsConfigured();
+ if (lane === 'openai') return Boolean(env.OPENAI_API_KEY);
+ return lane === 'edge';
+}
+
+/** The server-side TTS lanes that can serve right now, in failover order. */
+export function ttsLanes() {
+ return SERVER_LANES.filter(laneConfigured);
+}
+
+async function agentVoice(agentId) {
+ if (!agentId) return null;
+ const [row] = await sql`SELECT voice_provider, voice_id FROM agent_identities WHERE id = ${agentId}`.catch(() => []);
+ if (!row || !SERVER_LANES.includes(row.voice_provider)) return null;
+ return { provider: row.voice_provider, voiceId: row.voice_id || undefined };
}
/**
- * Encode speech for a chat voice note: Ogg/Opus, mono, 48 kHz, voice-tuned.
- * @param {Buffer} wav
- * @returns {Promise<{ buffer:Buffer, mimeType:string, durationSec:number|null }>}
+ * Speak a reply. Returns null when there is nothing to say; throws only when
+ * every lane failed, carrying the last lane's error.
+ * @returns {Promise<{ buffer:Buffer, mimeType:'audio/ogg', filename:string, provider:string } | null>}
*/
-export async function wavToVoiceNote(wav) {
- const buffer = await runFfmpeg(wav, ['-vn', '-ac', '1', '-ar', '48000', '-c:a', 'libopus', '-b:a', '32k', '-application', 'voip', '-f', 'ogg']);
- if (!buffer.length) throw new AudioError('transcode_failed', 'speech conversion produced no audio');
- return { buffer, mimeType: 'audio/ogg', durationSec: wavDurationSeconds(wav) };
+export async function synthesizeReply(reply, { agentId = null } = {}) {
+ const text = spokenText(reply);
+ if (!text) return null;
+ const preferred = await agentVoice(agentId);
+ const attempts = [];
+ if (preferred && laneConfigured(preferred.provider)) attempts.push(preferred);
+ for (const lane of ttsLanes()) if (lane !== preferred?.provider) attempts.push({ provider: lane });
+ let last = null;
+ for (const a of attempts) {
+ try {
+ const out = await synthesizeVoice({ provider: a.provider, voiceId: a.voiceId, text, format: a.provider === 'openai' ? 'opus' : undefined });
+ const buffer = await toOggOpus(out.audio, out.contentType);
+ return { buffer, mimeType: 'audio/ogg', filename: 'reply.ogg', provider: a.provider };
+ } catch (e) {
+ last = e;
+ console.warn(`[gateway] tts lane ${a.provider} failed`, e?.code || e?.message);
+ }
+ }
+ throw last || Object.assign(new Error('no speech lane is configured'), { code: 'not_configured' });
}
diff --git a/api/_lib/gateway/commands.js b/api/_lib/gateway/commands.js
index 9b140d123..e0521e5cc 100644
--- a/api/_lib/gateway/commands.js
+++ b/api/_lib/gateway/commands.js
@@ -1,6 +1,7 @@
-// Slash commands shared by Telegram (/balance) and Discord (/balance and
-// /three balance). Each reads the same data the web shows: the wallet card's
-// balance, the Portfolio tab, the agent's runs and its launch history.
+// Commands shared by every chat gateway: /balance on Telegram, WhatsApp, Signal
+// and SMS, /balance or /three balance on Discord, /three balance on Slack. Each
+// reads the same data the web shows: the wallet card's balance, the Portfolio
+// tab, the agent's runs and its launch history.
import { sql } from '../db.js';
import { readAgentSolBalance } from '../../agents/solana-wallet.js';
@@ -8,7 +9,8 @@ import { lamportsToUsd, listCustodyEvents } from '../agent-trade-guards.js';
import { getPortfolio } from '../portfolio.js';
import { queryAgentLaunches } from '../pump-agent-launches.js';
import { listAccountAgents, pickAgent, resolveChatAgent } from './agents.js';
-import { revokeLink, resetLinkContext, setLinkDefaultAgent } from './store.js';
+import { revokeLink, resetLinkContext, setLinkDefaultAgent, setLinkVoiceReplies } from './store.js';
+import { voiceRepliesAvailable } from './voice.js';
import { appOrigin, agentWalletUrl, explorerTx, fmtNum, fmtUsd, short } from './format.js';
export const COMMANDS = [
@@ -19,18 +21,27 @@ export const COMMANDS = [
{ name: 'agents', description: 'List your agents' },
{ name: 'use', description: 'Pick the agent this chat talks to', arg: 'agent' },
{ name: 'new', description: 'Start a fresh conversation thread' },
+ { name: 'voice', description: 'Turn spoken replies on or off', arg: 'on|off' },
{ name: 'unlink', description: 'Disconnect this chat from your account' },
{ name: 'link', description: 'Pair this chat with your three.ws account', arg: 'code' },
{ name: 'help', description: 'What this bot can do' },
];
-export function helpText() {
+/**
+ * @param {{ buttons?:boolean, prefix?:string }} [opts]
+ * buttons false on channels approved by reply code (SMS, Signal, email)
+ * prefix how a command is typed there: '/' everywhere except Slack ('/three ')
+ */
+export function helpText({ buttons = true, prefix = '/' } = {}) {
+ const approval = buttons
+ ? 'Trades never run from text alone. When your agent prepares one you get a preview with Approve and Cancel buttons; only Approve executes it, and a preview expires after ten minutes.'
+ : 'Trades never run from text alone. When your agent prepares one you get a preview ending with a six-digit code; only a reply of APPROVE and that exact code executes it, from this chat, within ten minutes.';
return [
'Talk to your three.ws agent here. Just write, send a voice note, or send a photo.',
'',
- ...COMMANDS.filter((c) => c.name !== 'link').map((c) => `/${c.name}${c.arg ? ` <${c.arg}>` : ''}: ${c.description}`),
+ ...COMMANDS.filter((c) => c.name !== 'link').map((c) => `${prefix}${c.name}${c.arg ? ` <${c.arg}>` : ''}: ${c.description}`),
'',
- 'Trades never run from text alone. When your agent prepares one you get a preview with Approve and Cancel buttons; only Approve executes it, and a preview expires after ten minutes.',
+ approval,
`Manage paired chats: ${appOrigin()}/settings/connections`,
].join('\n');
}
@@ -77,6 +88,19 @@ export async function cmdNew({ gw, event, link }) {
return gw.sendText(event.chatId, 'Started a fresh thread. Your agent will not see earlier messages here; the full history stays on the web.');
}
+export async function cmdVoice({ gw, event, link }) {
+ const arg = String(event.args || '').trim().toLowerCase();
+ if (typeof gw.sendVoice !== 'function') return gw.sendText(event.chatId, 'This channel cannot play voice notes, so spoken replies are not available here.');
+ if (!['on', 'off'].includes(arg)) {
+ return gw.sendText(event.chatId, `Spoken replies are ${link.voice_replies ? 'on' : 'off'} for this chat. Send /voice on or /voice off to change it.`);
+ }
+ if (arg === 'on' && !voiceRepliesAvailable()) return gw.sendText(event.chatId, 'Speech synthesis is not available right now, so spoken replies cannot be turned on. Try again later.');
+ await setLinkVoiceReplies(link.id, link.user_id, arg === 'on');
+ return gw.sendText(event.chatId, arg === 'on'
+ ? 'Spoken replies are on. Every answer arrives as text, then as a voice note.'
+ : 'Spoken replies are off. Answers arrive as text only.');
+}
+
export async function cmdBalance({ gw, event, link }) {
const agent = await needAgent(link, gw, event.chatId);
if (!agent) return;
@@ -168,6 +192,7 @@ export const COMMAND_HANDLERS = {
use: cmdUse,
unlink: cmdUnlink,
new: cmdNew,
+ voice: cmdVoice,
balance: cmdBalance,
portfolio: cmdPortfolio,
runs: cmdRuns,
diff --git a/api/_lib/gateway/conversation.js b/api/_lib/gateway/conversation.js
index 42a59eda4..74609693b 100644
--- a/api/_lib/gateway/conversation.js
+++ b/api/_lib/gateway/conversation.js
@@ -4,30 +4,45 @@
// over the agent's cross-channel thread (api/_lib/agent-thread.js), so a
// conversation started on the web continues here and the other way round. While
// the agent works, the chat shows a typing indicator and one compact status
-// line ("working: get_quote") edited in place. Every trade or limit change the
-// agent proposes becomes a preview with Approve and Cancel buttons; nothing in
-// the reply text can execute one.
+// line ("working: get_quote") edited in place, on platforms that can edit a
+// message (SMS, Signal and email cannot, so they get neither). Every trade or
+// limit change the agent proposes becomes a preview with Approve and Cancel
+// buttons, or on a channel without buttons a single-use reply code
+// (./reply-codes.js); nothing in the reply text can execute one. When the
+// owner turned on voice replies for the chat, the reply is also spoken
+// (./voice.js).
import { runCopilotTurn } from '../copilot-engine.js';
import { appendThreadMessage, threadHistoryForModel } from '../agent-thread.js';
import { resolveChatAgent, listAccountAgents } from './agents.js';
import { createPreview, setPreviewMessageRef, PREVIEW_TTL_MINUTES } from './store.js';
import { chunkText, describeProposal, plainText, MAX_TEXT, PLATFORM_LABEL, appOrigin } from './format.js';
+import { maybeSpeakReply } from './voice.js';
export const APPROVE = 'gw:ap:';
export const CANCEL = 'gw:cx:';
const TYPING_EVERY_MS = 4500;
-function surfaceNote(platform) {
+/** Whether a gateway renders buttons (true unless it says otherwise). */
+export function hasButtons(gw) {
+ return gw.buttons !== false;
+}
+
+function surfaceNote(platform, { buttons = true } = {}) {
const where = PLATFORM_LABEL[platform] || platform;
+ const approval = buttons
+ ? 'Every propose_* card you surface reaches the owner as a preview message with Approve and Cancel buttons. Tell them to press Approve to execute it or Cancel to discard it, and that it expires in ten minutes. Never claim a trade happened.'
+ : 'Every propose_* card you surface reaches the owner as a preview message ending with a six-digit code. Tell them to reply APPROVE and that code to execute it, or CANCEL and the code to discard it, and that it expires in ten minutes. Never claim a trade happened.';
+ const brevity = platform === 'sms' ? ' This is SMS: answer in two or three short sentences.' : ' Keep it short.';
return [
- `SURFACE: you are replying in a ${where} chat with your owner. Write plain text: no tables, no headings, no code blocks. Keep it short.`,
- 'Every propose_* card you surface reaches the owner as a preview message with Approve and Cancel buttons. Tell them to press Approve to execute it or Cancel to discard it, and that it expires in ten minutes. Never claim a trade happened.',
+ `SURFACE: you are replying in a ${where} chat with your owner. Write plain text: no tables, no headings, no code blocks.${brevity}`,
+ approval,
].join('\n');
}
/** Keep the platform's typing indicator alive until stop() is called. */
function keepTyping(gw, chatId) {
+ if (typeof gw.typing !== 'function') return () => {};
let stopped = false;
const tick = () => { if (!stopped) Promise.resolve(gw.typing(chatId)).catch(() => {}); };
tick();
@@ -40,6 +55,9 @@ function keepTyping(gw, chatId) {
* left behind as a compact record of what the agent looked at.
*/
function statusLine(gw, chatId) {
+ // A channel that cannot edit would get one new message per tool call; on
+ // SMS that is a bill, so those channels skip the status line entirely.
+ if (gw.canEdit === false) return { working() {}, async finish() {} };
let ref = null;
let chain = Promise.resolve();
const used = [];
@@ -82,7 +100,7 @@ export function previewText(proposal) {
async function sendPreviews({ gw, event, link, agent, proposals }) {
for (const p of proposals) {
const preview = await createPreview({ linkId: link.id, userId: link.user_id, agentId: agent.id, kind: p.kind, proposal: p });
- const ref = await gw.sendChoice(event.chatId, previewText(p), choicesFor(preview, p));
+ const ref = await gw.sendChoice(event.chatId, previewText(p), choicesFor(preview, p), { link });
if (ref) await setPreviewMessageRef(preview.id, ref);
}
}
@@ -117,7 +135,7 @@ export async function converse({ gw, event, link, text }) {
agent,
history,
network: 'mainnet',
- surfaceNote: surfaceNote(event.platform),
+ surfaceNote: surfaceNote(event.platform, { buttons: hasButtons(gw) }),
emit: (name, data) => { if (name === 'tool_start') status.working(data.name); },
});
} catch (e) {
@@ -134,6 +152,7 @@ export async function converse({ gw, event, link, text }) {
const reply = plainText(turn.reply) || (turn.proposals.length ? 'Here is what I prepared:' : 'I have nothing to add to that.');
await sendReply(gw, event.chatId, reply);
+ await maybeSpeakReply({ gw, chatId: event.chatId, link, text: reply, agentId: agent.id });
if (turn.proposals.length) await sendPreviews({ gw, event, link, agent, proposals: turn.proposals });
await appendThreadMessage({
diff --git a/api/_lib/gateway/media.js b/api/_lib/gateway/media.js
index 69fb4b1ea..305e7b52e 100644
--- a/api/_lib/gateway/media.js
+++ b/api/_lib/gateway/media.js
@@ -1,10 +1,10 @@
// Voice notes and photos sent to a chat gateway become text the agent can read.
-// Voice runs through the same NVIDIA Riva speech path as POST /api/asr; photos
-// run through the same vision chain as POST /api/vision.
+// Voice runs through the gateway audio path (./audio.js: the NVIDIA Riva lane
+// behind POST /api/asr, Gemini on Vertex as the failover, any container
+// transcoded first); photos run through the same vision chain as POST /api/vision.
-import { transcribeNvidiaAsr, nvidiaAsrConfigured } from '../asr-nvidia.js';
import { describeImage, visionConfigured } from '../vision.js';
-import { toSpeechInput, AudioError } from './audio.js';
+import { transcribeAudio, speechConfigured } from './audio.js';
export const MAX_VOICE_BYTES = 8 * 1024 * 1024;
export const MAX_IMAGE_BYTES = 8 * 1024 * 1024;
@@ -16,25 +16,23 @@ export class MediaError extends Error {
}
}
+const UNREADABLE = new Set(['transcode_failed', 'transcode_timeout']);
+
/**
- * Transcribe a chat voice note. Ogg/Opus (Telegram, Discord, WhatsApp) and FLAC
- * go to Riva as-is; AAC, MP4 and WebM (Signal, iOS, Slack clips) are converted
- * to 16 kHz PCM first (./audio.js).
+ * Transcribe a chat voice note in whatever container the platform recorded it.
* @returns {Promise<string>}
*/
export async function transcribeVoice({ buffer, mimeType = 'audio/ogg', language = 'en-US' }) {
- if (!nvidiaAsrConfigured()) throw new MediaError('asr_unavailable', 'Voice notes are not available right now. Type your message instead.');
+ if (!speechConfigured()) throw new MediaError('asr_unavailable', 'Voice notes are not available right now. Type your message instead.');
if (!buffer?.length) throw new MediaError('empty_audio', 'That voice note was empty.');
if (buffer.length > MAX_VOICE_BYTES) throw new MediaError('audio_too_large', 'That voice note is too long. Keep it under about five minutes.');
- let input;
+ let text;
try {
- input = await toSpeechInput({ buffer, mimeType });
+ text = await transcribeAudio({ buffer, mimeType, language });
} catch (e) {
- if (e instanceof AudioError) throw new MediaError('audio_unreadable', 'I could not read that voice note. Try recording it again, or type it.');
+ if (UNREADABLE.has(e?.code)) throw new MediaError('audio_unreadable', 'I could not read that voice note. Try recording it again, or type it.');
throw e;
}
- const out = await transcribeNvidiaAsr({ audio: input.audio, encoding: input.encoding, sampleRateHz: input.sampleRateHz, language });
- const text = String(out?.text || '').trim();
if (!text) throw new MediaError('no_speech', 'I could not make out any words in that voice note.');
return text;
}
diff --git a/api/_lib/gateway/voice.js b/api/_lib/gateway/voice.js
new file mode 100644
index 000000000..a1b4d889a
--- /dev/null
+++ b/api/_lib/gateway/voice.js
@@ -0,0 +1,32 @@
+// Spoken replies for chats with "voice replies" turned on (/voice on in the
+// chat, or the toggle on /settings/connections). After the text reply lands,
+// the same reply is rendered as a voice note (./audio.js synthesizeReply) and
+// sent with the adapter's sendVoice. A channel whose adapter has no sendVoice
+// (SMS) never offers the setting. A speech failure never fails the turn: the
+// owner already has the text.
+
+import { synthesizeReply, ttsLanes } from './audio.js';
+
+/** Whether any TTS lane can speak right now. */
+export function voiceRepliesAvailable() {
+ return ttsLanes().length > 0;
+}
+
+/**
+ * Speak `text` into the chat when the link asked for it and the platform can
+ * play audio. Returns what happened, for logs and tests.
+ * @returns {Promise<'sent'|'off'|'unsupported'|'empty'|'failed'>}
+ */
+export async function maybeSpeakReply({ gw, chatId, link, text, agentId = null }) {
+ if (!link?.voice_replies) return 'off';
+ if (typeof gw.sendVoice !== 'function') return 'unsupported';
+ try {
+ const clip = await synthesizeReply(text, { agentId: agentId || link.default_agent_id || null });
+ if (!clip) return 'empty';
+ await gw.sendVoice(chatId, clip);
+ return 'sent';
+ } catch (e) {
+ console.warn('[gateway] spoken reply failed', e?.code || e?.message);
+ return 'failed';
+ }
+}
diff --git a/api/_lib/gateway/webhooks.js b/api/_lib/gateway/webhooks.js
index fc8782f8d..0f3c573b8 100644
--- a/api/_lib/gateway/webhooks.js
+++ b/api/_lib/gateway/webhooks.js
@@ -1,9 +1,9 @@
// Verification and normalisation for the chat gateway webhooks. The receivers
-// (api/gateway/telegram.js, api/gateway/discord.js) verify the platform's proof,
+// (api/gateway/{telegram,discord,slack,whatsapp,sms}.js) verify the platform's proof,
// queue the delivery in gateway_inbox and answer at once; the worker does the
// slow part. Kept free of HTTP so the tests can drive every branch.
-import { createHash, createPublicKey, timingSafeEqual, verify } from 'node:crypto';
+import { createHash, createHmac, createPublicKey, timingSafeEqual, verify } from 'node:crypto';
// ── Telegram ─────────────────────────────────────────────────────────────────
@@ -89,3 +89,132 @@ export function discordInboxKeys(interaction) {
if (interaction.type !== DISCORD_INTERACTION.APPLICATION_COMMAND && interaction.type !== DISCORD_INTERACTION.MESSAGE_COMPONENT) return null;
return { dedupeKey: String(interaction.id), chatKey: `discord:${interaction.channel_id}` };
}
+
+// ── Slack ────────────────────────────────────────────────────────────────────
+
+const SLACK_MAX_SKEW_S = 300;
+
+/**
+ * Verify Slack's v0 request signature: HMAC-SHA256 of `v0:<timestamp>:<raw body>`
+ * keyed with the app's signing secret, sent as `v0=<hex>` in X-Slack-Signature.
+ * A timestamp more than five minutes off is refused so a capture cannot be replayed.
+ */
+export function verifySlackSignature({ rawBody, signature, timestamp, signingSecret, now = Date.now() }) {
+ if (!signingSecret || !rawBody) return false;
+ if (!/^\d{1,12}$/.test(String(timestamp || ''))) return false;
+ if (Math.abs(now / 1000 - Number(timestamp)) > SLACK_MAX_SKEW_S) return false;
+ const body = Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(String(rawBody));
+ const expected = `v0=${createHmac('sha256', signingSecret).update(`v0:${timestamp}:`).update(body).digest('hex')}`;
+ return safeEqual(signature, expected);
+}
+
+/**
+ * Classify a verified Slack delivery. The one endpoint receives three shapes:
+ * Events API JSON, slash commands (form fields) and interactivity (a form field
+ * `payload` holding JSON). Returns what the receiver should do with it.
+ * @param {{ contentType:string, rawBody:Buffer|string }} req
+ * @returns {{ type:'challenge', challenge:string }
+ * | { type:'enqueue', dedupeKey:string, chatKey:string, payload:object }
+ * | { type:'ignore' }}
+ */
+export function classifySlackDelivery({ contentType, rawBody }) {
+ const text = Buffer.isBuffer(rawBody) ? rawBody.toString('utf8') : String(rawBody || '');
+ if (/application\/json/i.test(contentType || '')) {
+ let body;
+ try { body = JSON.parse(text); } catch { return { type: 'ignore' }; }
+ if (body?.type === 'url_verification' && typeof body.challenge === 'string') return { type: 'challenge', challenge: body.challenge };
+ if (body?.type !== 'event_callback' || !body.event) return { type: 'ignore' };
+ const ev = body.event;
+ // The bot's own posts, edits and joins come back as events too.
+ if (ev.bot_id || ev.subtype) return { type: 'ignore' };
+ const direct = ev.type === 'message' && ev.channel_type === 'im';
+ if (!direct && ev.type !== 'app_mention') return { type: 'ignore' };
+ if (!ev.channel || !ev.user) return { type: 'ignore' };
+ return { type: 'enqueue', dedupeKey: `event:${body.event_id || ev.client_msg_id || ev.ts}`, chatKey: `slack:${ev.channel}`, payload: { kind: 'slack_event', team_id: body.team_id, event: ev } };
+ }
+ const form = Object.fromEntries(new URLSearchParams(text));
+ if (form.payload) {
+ let p;
+ try { p = JSON.parse(form.payload); } catch { return { type: 'ignore' }; }
+ if (p?.type !== 'block_actions' || !Array.isArray(p.actions) || !p.actions.length) return { type: 'ignore' };
+ const channel = p.channel?.id || p.container?.channel_id;
+ if (!channel) return { type: 'ignore' };
+ const a = p.actions[0];
+ return { type: 'enqueue', dedupeKey: `action:${p.trigger_id || `${a.action_id}:${a.action_ts}`}`, chatKey: `slack:${channel}`, payload: { kind: 'slack_action', action: p } };
+ }
+ if (form.command && form.channel_id && form.user_id) {
+ return { type: 'enqueue', dedupeKey: `command:${form.trigger_id || `${form.channel_id}:${form.user_id}:${Date.now()}`}`, chatKey: `slack:${form.channel_id}`, payload: { kind: 'slack_command', command: form } };
+ }
+ return { type: 'ignore' };
+}
+
+// ── WhatsApp (Cloud API) ─────────────────────────────────────────────────────
+
+/**
+ * Verify Meta's X-Hub-Signature-256: `sha256=` + HMAC-SHA256 of the raw body
+ * keyed with the app secret.
+ */
+export function verifyWhatsAppSignature({ rawBody, signature, appSecret }) {
+ if (!appSecret || !rawBody) return false;
+ const body = Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(String(rawBody));
+ const expected = `sha256=${createHmac('sha256', appSecret).update(body).digest('hex')}`;
+ return safeEqual(String(signature || '').toLowerCase(), expected);
+}
+
+/**
+ * Answer Meta's subscription handshake (GET ?hub.mode=subscribe&hub.verify_token=
+ * &hub.challenge=). Returns the challenge to echo, or null to refuse.
+ */
+export function whatsappChallenge(query, verifyToken) {
+ if (!verifyToken) return null;
+ if (query?.['hub.mode'] !== 'subscribe') return null;
+ if (!safeEqual(query?.['hub.verify_token'], verifyToken)) return null;
+ const challenge = String(query?.['hub.challenge'] || '');
+ return /^[\w-]{1,128}$/.test(challenge) ? challenge : null;
+}
+
+/**
+ * One Cloud API webhook can batch several messages (and status receipts, which
+ * the gateway ignores). Each message becomes its own inbox row.
+ * @returns {Array<{ dedupeKey:string, chatKey:string, payload:object }>}
+ */
+export function whatsappInboxEntries(body) {
+ const out = [];
+ if (body?.object !== 'whatsapp_business_account') return out;
+ for (const entry of body.entry || []) {
+ for (const change of entry.changes || []) {
+ const value = change?.value;
+ if (change?.field !== 'messages' || !value) continue;
+ for (const message of value.messages || []) {
+ if (!message?.id || !message.from) continue;
+ const contact = (value.contacts || []).find((c) => c.wa_id === message.from) || null;
+ out.push({
+ dedupeKey: String(message.id),
+ chatKey: `whatsapp:${message.from}`,
+ payload: { kind: 'whatsapp_message', phone_number_id: value.metadata?.phone_number_id || null, contact, message },
+ });
+ }
+ }
+ }
+ return out;
+}
+
+// ── SMS (Twilio) ─────────────────────────────────────────────────────────────
+
+/**
+ * Verify X-Twilio-Signature: base64 HMAC-SHA1, keyed with the auth token, of the
+ * exact public URL Twilio posted to followed by every form field as key+value,
+ * keys sorted.
+ */
+export function verifyTwilioSignature({ url, params, signature, authToken }) {
+ if (!authToken || !url || !signature) return false;
+ const data = Object.keys(params || {}).sort().reduce((s, k) => s + k + String(params[k] ?? ''), String(url));
+ const expected = createHmac('sha1', authToken).update(Buffer.from(data, 'utf8')).digest('base64');
+ return safeEqual(signature, expected);
+}
+
+/** @returns {{ dedupeKey:string, chatKey:string } | null} */
+export function smsInboxKeys(params) {
+ if (!params?.MessageSid || !params.From) return null;
+ return { dedupeKey: String(params.MessageSid), chatKey: `sms:${params.From}` };
+}