Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(gateway): spoken replies, reply-code friendly approvals and Slack, WhatsApp and SMS webhook verification
nirholas66d378154407
diff --git a/api/_lib/gateway/approvals.js b/api/_lib/gateway/approvals.jsindex c124bf6dd..828f374b2 100644--- a/api/_lib/gateway/approvals.js+++ b/api/_lib/gateway/approvals.js@@ -126,11 +126,21 @@ export async function handleAction({ gw, event, link }) {}const ref = preview.message_ref || event.messageRef || null;const base = previewText(preview.proposal);- const settle = async (line) => { if (ref) await gw.editMessage(ref, `${base}\n\n${line}`, { choices: [] }).catch(() => {}); };+ // A channel that can edit rewrites the preview in place and drops its+ // buttons. One that cannot (SMS, Signal, email: approved by reply code)+ // gets the outcome as a new message, and skips the transient acks that+ // would otherwise arrive as a second, redundant text.+ const editable = gw.canEdit !== false;+ const settle = async (line) => {+ if (!editable) return gw.sendText(event.chatId, line).catch(() => {});+ if (ref) await gw.editMessage(ref, `${base}\n\n${line}`, { choices: [] }).catch(() => {});+ };+ const ackThenSettle = (text) => (editable ? gw.ackAction(event, text) : null);if (verb === 'cancel') {const done = await cancelPreview(preview.id);- await gw.ackAction(event, done ? 'Cancelled' : `Already ${preview.status}`);+ if (done) await ackThenSettle('Cancelled');+ else await gw.ackAction(event, `Already ${preview.status}`);if (done) {await settle('Cancelled. Nothing was sent.');await appendThreadMessage({ agentId: preview.agent_id, userId: preview.user_id, role: 'assistant', content: `Cancelled the ${preview.kind} preview. Nothing was sent.`, channel: event.platform }).catch(() => {});@@ -141,7 +151,7 @@ export async function handleAction({ gw, event, link }) {if (preview.status !== 'pending') return gw.ackAction(event, `Already ${preview.status}.`);if (new Date(preview.expires_at) <= new Date()) {await expirePreview(preview.id);- await gw.ackAction(event, 'This preview expired.');+ await ackThenSettle('This preview expired.');return settle('Expired. Ask your agent again for a fresh quote.');}const rl = await limits.gatewayMessage(link.id);@@ -149,7 +159,7 @@ export async function handleAction({ gw, event, link }) {const claimed = await claimPreview(preview.id);if (!claimed) return gw.ackAction(event, 'This preview was already handled.');- await gw.ackAction(event, 'Approved. Executing...');+ await ackThenSettle('Approved. Executing...');await settle('Approved. Executing...');let result;diff --git a/api/_lib/gateway/audio.js b/api/_lib/gateway/audio.jsindex 4601277b7..d23568ce5 100644--- a/api/_lib/gateway/audio.js+++ b/api/_lib/gateway/audio.js@@ -1,128 +1,197 @@-// Audio plumbing for the chat gateways: every platform delivers voice notes in-// its own container (Telegram, WhatsApp and Discord send Ogg/Opus, Signal and-// iOS clients send AAC in MP4, Slack clips arrive as WebM or MP4), and every-// platform that plays a voice note wants Ogg/Opus back. The speech lanes speak-// WAV and listen to Ogg/Opus, FLAC or 16-bit PCM, so this module converts at-// the edges with ffmpeg and nothing else in the gateway touches a codec.+// Audio for the chat gateways: turning whatever a platform delivers into+// something the speech lane can hear, and turning an agent's reply into a voice+// note every platform can play.//-// The ffmpeg binary comes from FFMPEG_PATH (the gateway image installs the-// distro build there), then the ffmpeg-static package, then the PATH.+// Inbound: Telegram, Discord and WhatsApp send Ogg/Opus, which the NVIDIA Riva+// lane (the same one behind POST /api/asr) accepts as is. Slack clips arrive as+// WebM or MP4, Signal as AAC and MMS as AMR, so those are transcoded to Ogg/Opus+// with ffmpeg first. When Riva is unreachable or unconfigured, the clip goes to+// Gemini on Vertex AI (platform GCP credits) instead, so a voice note still+// lands as text.+//+// Outbound: a reply is synthesized on the platform's server-side TTS lanes+// (api/_lib/voice-providers.js synthesizeVoice), starting with the agent's own+// configured voice when it lives on one of those lanes, then NVIDIA Magpie,+// Gemini on Vertex, the free Edge lane and OpenAI. The clip is always delivered+// as a 48 kHz mono Ogg/Opus voice note: the one container Telegram shows as a+// voice bubble, WhatsApp as a voice message, and Signal, Slack and Discord play.import { spawn } from 'node:child_process';+import { createRequire } from 'node:module';+import { sql } from '../db.js';+import { env } from '../env.js';+import { transcribeNvidiaAsr, nvidiaAsrConfigured } from '../asr-nvidia.js';+import { synthesizeVoice } from '../voice-providers.js';+import { geminiTtsConfigured } from '../tts-gemini.js';+import { nvidiaTtsConfigured } from '../tts-nvidia.js';+import { getGcpAccessToken } from '../gcp-auth.js';-let resolvedPath = null;+export const SPOKEN_MAX_CHARS = 700;+const FFMPEG_TIMEOUT_MS = 30_000;+const SERVER_LANES = ['nvidia', 'gemini', 'edge', 'openai'];+const OPUS_ARGS = ['-vn', '-ac', '1', '-ar', '48000', '-c:a', 'libopus', '-b:a', '32k', '-application', 'voip', '-f', 'ogg'];-export async function ffmpegPath() {- if (resolvedPath) return resolvedPath;- if (process.env.FFMPEG_PATH) return (resolvedPath = process.env.FFMPEG_PATH);+/**+ * The ffmpeg binary: FFMPEG_PATH when set (the gateway image installs the+ * distro ffmpeg), the ffmpeg-static binary when its install script ran, else+ * `ffmpeg` on PATH.+ */+export function ffmpegPath() {+ if (process.env.FFMPEG_PATH) return process.env.FFMPEG_PATH;try {- const mod = await import('ffmpeg-static');- if (mod?.default) return (resolvedPath = mod.default);+ const p = createRequire(import.meta.url)('ffmpeg-static');+ if (typeof p === 'string' && p) return p;} catch {- // Not installed in this image: fall through to the PATH.- }- return (resolvedPath = 'ffmpeg');-}--export class AudioError extends Error {- constructor(code, message) {- super(message);- this.code = code;+ // ffmpeg-static is optional here: the PATH binary below serves instead.}+ return 'ffmpeg';}-/**- * Pipe `input` through ffmpeg with `args` (input is always stdin, output is- * always stdout) and resolve with the output bytes.- * @param {Buffer} input- * @param {string[]} args arguments between `-i pipe:0` and `pipe:1`- * @param {{ timeoutMs?:number, maxBytes?:number }} [opts]- * @returns {Promise<Buffer>}- */-export async function runFfmpeg(input, args, { timeoutMs = 30_000, maxBytes = 32 * 1024 * 1024 } = {}) {- const bin = await ffmpegPath();+/** Run ffmpeg over a buffer: stdin in, stdout out. */+export function runFfmpeg(input, args, { timeoutMs = FFMPEG_TIMEOUT_MS } = {}) {return new Promise((resolve, reject) => {- const proc = spawn(bin, ['-hide_banner', '-loglevel', 'error', '-i', 'pipe:0', ...args, 'pipe:1'], { stdio: ['pipe', 'pipe', 'pipe'] });+ const proc = spawn(ffmpegPath(), ['-hide_banner', '-loglevel', 'error', '-i', 'pipe:0', ...args, 'pipe:1']);const out = [];- let size = 0;- let stderr = '';- const timer = setTimeout(() => {- proc.kill('SIGKILL');- reject(new AudioError('transcode_timeout', 'audio conversion took too long'));- }, timeoutMs);- proc.on('error', (e) => {- clearTimeout(timer);- reject(new AudioError('transcoder_unavailable', `ffmpeg could not start: ${e.message}`));- });- proc.stdout.on('data', (chunk) => {- size += chunk.length;- if (size > maxBytes) {- proc.kill('SIGKILL');- return;- }- out.push(chunk);- });- proc.stderr.on('data', (d) => { stderr = (stderr + d.toString()).slice(-600); });+ let err = '';+ const timer = setTimeout(() => { proc.kill('SIGKILL'); reject(Object.assign(new Error('ffmpeg timed out'), { code: 'transcode_timeout' })); }, timeoutMs);+ proc.stdout.on('data', (d) => out.push(d));+ proc.stderr.on('data', (d) => { err += d; });+ proc.on('error', (e) => { clearTimeout(timer); reject(Object.assign(e, { code: 'transcode_unavailable' })); });proc.on('close', (code) => {clearTimeout(timer);- if (size > maxBytes) return reject(new AudioError('audio_too_large', 'converted audio exceeded the size limit'));- if (code !== 0) return reject(new AudioError('transcode_failed', `ffmpeg exited ${code}: ${stderr.trim()}`));- resolve(Buffer.concat(out));+ if (code === 0 && out.length) resolve(Buffer.concat(out));+ else reject(Object.assign(new Error(`ffmpeg exited ${code}: ${err.trim().slice(0, 300)}`), { code: 'transcode_failed' }));});- // A client that closes stdin early (bad container) must not crash the worker.proc.stdin.on('error', () => {});proc.stdin.end(input);});}-/** The speech-lane encoding a container can be sent as without conversion, or null. */-export function nativeAsrEncoding(mimeType) {- const m = String(mimeType || '').toLowerCase();- if (/ogg|opus/.test(m)) return 'OGGOPUS';- if (/flac/.test(m)) return 'FLAC';- return null;+/** True when a clip is already Ogg/Opus and Riva can take it untouched. */+export function isOggOpus(mimeType) {+ return /^audio\/(ogg|opus)/i.test(String(mimeType || ''));}-/**- * Turn any voice note into something the speech lane accepts: Ogg/Opus and- * FLAC pass through, everything else becomes 16 kHz mono 16-bit PCM.- * @returns {Promise<{ audio:Buffer, encoding:string, sampleRateHz:number }>}- */-export async function toSpeechInput({ buffer, mimeType }) {- const native = nativeAsrEncoding(mimeType);- if (native === 'OGGOPUS') return { audio: buffer, encoding: 'OGGOPUS', sampleRateHz: 48000 };- if (native === 'FLAC') return { audio: buffer, encoding: 'FLAC', sampleRateHz: 16000 };- const pcm = await runFfmpeg(buffer, ['-vn', '-ac', '1', '-ar', '16000', '-f', 's16le', '-acodec', 'pcm_s16le']);- if (!pcm.length) throw new AudioError('no_audio', 'that voice note carried no audio');- return { audio: pcm, encoding: 'LINEAR_PCM', sampleRateHz: 16000 };+/** Any recorded audio as mono 48 kHz Ogg/Opus. Ogg input passes through untouched. */+export async function toOggOpus(buffer, mimeType = null) {+ if (isOggOpus(mimeType)) return buffer;+ return runFfmpeg(buffer, OPUS_ARGS);+}++async function transcribeWithGemini(ogg) {+ const project = process.env.GOOGLE_CLOUD_PROJECT;+ if (!project) throw Object.assign(new Error('Vertex AI is not configured'), { code: 'not_configured' });+ const location = process.env.GOOGLE_CLOUD_LOCATION_GEMINI || 'global';+ const host = location === 'global' ? 'aiplatform.googleapis.com' : `${location}-aiplatform.googleapis.com`;+ const model = (process.env.VERTEX_GEMINI_MODEL || 'google/gemini-2.5-flash').replace(/^google\//, '');+ const token = await getGcpAccessToken();+ const r = await fetch(`https://${host}/v1/projects/${project}/locations/${location}/publishers/google/models/${model}:generateContent`, {+ method: 'POST',+ headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },+ body: JSON.stringify({+ contents: [{+ role: 'user',+ parts: [+ { inlineData: { mimeType: 'audio/ogg', data: ogg.toString('base64') } },+ { text: 'Transcribe this voice note verbatim. Reply with the transcript only, no commentary. If there is no speech, reply with nothing.' },+ ],+ }],+ generationConfig: { temperature: 0, maxOutputTokens: 1024, thinkingConfig: { thinkingBudget: 0 } },+ }),+ signal: AbortSignal.timeout(30_000),+ });+ const j = await r.json().catch(() => null);+ if (!r.ok) throw Object.assign(new Error(j?.error?.message || `vertex ${r.status}`), { code: 'provider_error' });+ return (j?.candidates?.[0]?.content?.parts || []).map((p) => p.text || '').join('').trim();}-/** Duration in seconds of a canonical RIFF/WAVE buffer, or null when it is not one. */-export function wavDurationSeconds(wav) {- if (!Buffer.isBuffer(wav) || wav.length < 44 || wav.toString('ascii', 0, 4) !== 'RIFF' || wav.toString('ascii', 8, 12) !== 'WAVE') return null;- let offset = 12;- let byteRate = null;- while (offset + 8 <= wav.length) {- const id = wav.toString('ascii', offset, offset + 4);- const size = wav.readUInt32LE(offset + 4);- if (id === 'fmt ') byteRate = wav.readUInt32LE(offset + 16);- if (id === 'data') {- if (!byteRate) return null;- const dataBytes = Math.min(size, wav.length - offset - 8);- return dataBytes / byteRate;+/** Whether any speech-to-text lane can serve. */+export function speechConfigured() {+ return nvidiaAsrConfigured() || Boolean(process.env.GOOGLE_CLOUD_PROJECT);+}++/**+ * Transcribe a voice note in any container. Riva first; Gemini on Vertex when+ * Riva is down or unconfigured.+ * @returns {Promise<string>} the transcript (empty when nothing was said)+ */+export async function transcribeAudio({ buffer, mimeType, language = 'en-US' }) {+ const ogg = await toOggOpus(buffer, mimeType);+ if (nvidiaAsrConfigured()) {+ try {+ const out = await transcribeNvidiaAsr({ audio: ogg, encoding: 'OGGOPUS', sampleRateHz: 48000, language });+ return String(out?.text || '').trim();+ } catch (e) {+ if (!process.env.GOOGLE_CLOUD_PROJECT) throw e;+ console.warn('[gateway] riva failed, trying gemini', e?.code || e?.message);}- offset += 8 + size + (size % 2);}- return null;+ return transcribeWithGemini(ogg);+}++/**+ * The part of a reply worth saying out loud: markdown, links and long base58+ * strings (addresses, signatures) removed, cut at a sentence boundary so the+ * clip never stops mid-word.+ */+export function spokenText(reply, max = SPOKEN_MAX_CHARS) {+ let s = String(reply || '')+ .replace(/```[\s\S]*?```/g, ' ')+ .replace(/\[([^\]]+)\]\((?:https?:[^)]+)\)/g, '$1')+ .replace(/https?:\/\/\S+/g, '')+ .replace(/\b[1-9A-HJ-NP-Za-km-z]{32,88}\b/g, '')+ .replace(/[*_`#>]/g, '')+ .replace(/\n{2,}/g, '. ')+ .replace(/\s+/g, ' ')+ .replace(/\s+([.,!?])/g, '$1')+ .trim();+ if (s.length <= max) return s;+ s = s.slice(0, max);+ const cut = Math.max(s.lastIndexOf('. '), s.lastIndexOf('! '), s.lastIndexOf('? '));+ return (cut > max * 0.4 ? s.slice(0, cut + 1) : `${s.slice(0, s.lastIndexOf(' '))}.`).trim();+}++function laneConfigured(lane) {+ if (lane === 'nvidia') return nvidiaTtsConfigured();+ if (lane === 'gemini') return geminiTtsConfigured();+ if (lane === 'openai') return Boolean(env.OPENAI_API_KEY);+ return lane === 'edge';+}++/** The server-side TTS lanes that can serve right now, in failover order. */+export function ttsLanes() {+ return SERVER_LANES.filter(laneConfigured);+}++async function agentVoice(agentId) {+ if (!agentId) return null;+ const [row] = await sql`SELECT voice_provider, voice_id FROM agent_identities WHERE id = ${agentId}`.catch(() => []);+ if (!row || !SERVER_LANES.includes(row.voice_provider)) return null;+ return { provider: row.voice_provider, voiceId: row.voice_id || undefined };}/**- * Encode speech for a chat voice note: Ogg/Opus, mono, 48 kHz, voice-tuned.- * @param {Buffer} wav- * @returns {Promise<{ buffer:Buffer, mimeType:string, durationSec:number|null }>}+ * Speak a reply. Returns null when there is nothing to say; throws only when+ * every lane failed, carrying the last lane's error.+ * @returns {Promise<{ buffer:Buffer, mimeType:'audio/ogg', filename:string, provider:string } | null>}*/-export async function wavToVoiceNote(wav) {- const buffer = await runFfmpeg(wav, ['-vn', '-ac', '1', '-ar', '48000', '-c:a', 'libopus', '-b:a', '32k', '-application', 'voip', '-f', 'ogg']);- if (!buffer.length) throw new AudioError('transcode_failed', 'speech conversion produced no audio');- return { buffer, mimeType: 'audio/ogg', durationSec: wavDurationSeconds(wav) };+export async function synthesizeReply(reply, { agentId = null } = {}) {+ const text = spokenText(reply);+ if (!text) return null;+ const preferred = await agentVoice(agentId);+ const attempts = [];+ if (preferred && laneConfigured(preferred.provider)) attempts.push(preferred);+ for (const lane of ttsLanes()) if (lane !== preferred?.provider) attempts.push({ provider: lane });+ let last = null;+ for (const a of attempts) {+ try {+ const out = await synthesizeVoice({ provider: a.provider, voiceId: a.voiceId, text, format: a.provider === 'openai' ? 'opus' : undefined });+ const buffer = await toOggOpus(out.audio, out.contentType);+ return { buffer, mimeType: 'audio/ogg', filename: 'reply.ogg', provider: a.provider };+ } catch (e) {+ last = e;+ console.warn(`[gateway] tts lane ${a.provider} failed`, e?.code || e?.message);+ }+ }+ throw last || Object.assign(new Error('no speech lane is configured'), { code: 'not_configured' });}diff --git a/api/_lib/gateway/commands.js b/api/_lib/gateway/commands.jsindex 9b140d123..e0521e5cc 100644--- a/api/_lib/gateway/commands.js+++ b/api/_lib/gateway/commands.js@@ -1,6 +1,7 @@-// Slash commands shared by Telegram (/balance) and Discord (/balance and-// /three balance). Each reads the same data the web shows: the wallet card's-// balance, the Portfolio tab, the agent's runs and its launch history.+// Commands shared by every chat gateway: /balance on Telegram, WhatsApp, Signal+// and SMS, /balance or /three balance on Discord, /three balance on Slack. Each+// reads the same data the web shows: the wallet card's balance, the Portfolio+// tab, the agent's runs and its launch history.import { sql } from '../db.js';import { readAgentSolBalance } from '../../agents/solana-wallet.js';@@ -8,7 +9,8 @@ import { lamportsToUsd, listCustodyEvents } from '../agent-trade-guards.js';import { getPortfolio } from '../portfolio.js';import { queryAgentLaunches } from '../pump-agent-launches.js';import { listAccountAgents, pickAgent, resolveChatAgent } from './agents.js';-import { revokeLink, resetLinkContext, setLinkDefaultAgent } from './store.js';+import { revokeLink, resetLinkContext, setLinkDefaultAgent, setLinkVoiceReplies } from './store.js';+import { voiceRepliesAvailable } from './voice.js';import { appOrigin, agentWalletUrl, explorerTx, fmtNum, fmtUsd, short } from './format.js';export const COMMANDS = [@@ -19,18 +21,27 @@ export const COMMANDS = [{ name: 'agents', description: 'List your agents' },{ name: 'use', description: 'Pick the agent this chat talks to', arg: 'agent' },{ name: 'new', description: 'Start a fresh conversation thread' },+ { name: 'voice', description: 'Turn spoken replies on or off', arg: 'on|off' },{ name: 'unlink', description: 'Disconnect this chat from your account' },{ name: 'link', description: 'Pair this chat with your three.ws account', arg: 'code' },{ name: 'help', description: 'What this bot can do' },];-export function helpText() {+/**+ * @param {{ buttons?:boolean, prefix?:string }} [opts]+ * buttons false on channels approved by reply code (SMS, Signal, email)+ * prefix how a command is typed there: '/' everywhere except Slack ('/three ')+ */+export function helpText({ buttons = true, prefix = '/' } = {}) {+ const approval = buttons+ ? 'Trades never run from text alone. When your agent prepares one you get a preview with Approve and Cancel buttons; only Approve executes it, and a preview expires after ten minutes.'+ : 'Trades never run from text alone. When your agent prepares one you get a preview ending with a six-digit code; only a reply of APPROVE and that exact code executes it, from this chat, within ten minutes.';return ['Talk to your three.ws agent here. Just write, send a voice note, or send a photo.','',- ...COMMANDS.filter((c) => c.name !== 'link').map((c) => `/${c.name}${c.arg ? ` <${c.arg}>` : ''}: ${c.description}`),+ ...COMMANDS.filter((c) => c.name !== 'link').map((c) => `${prefix}${c.name}${c.arg ? ` <${c.arg}>` : ''}: ${c.description}`),'',- 'Trades never run from text alone. When your agent prepares one you get a preview with Approve and Cancel buttons; only Approve executes it, and a preview expires after ten minutes.',+ approval,`Manage paired chats: ${appOrigin()}/settings/connections`,].join('\n');}@@ -77,6 +88,19 @@ export async function cmdNew({ gw, event, link }) {return gw.sendText(event.chatId, 'Started a fresh thread. Your agent will not see earlier messages here; the full history stays on the web.');}+export async function cmdVoice({ gw, event, link }) {+ const arg = String(event.args || '').trim().toLowerCase();+ if (typeof gw.sendVoice !== 'function') return gw.sendText(event.chatId, 'This channel cannot play voice notes, so spoken replies are not available here.');+ if (!['on', 'off'].includes(arg)) {+ return gw.sendText(event.chatId, `Spoken replies are ${link.voice_replies ? 'on' : 'off'} for this chat. Send /voice on or /voice off to change it.`);+ }+ if (arg === 'on' && !voiceRepliesAvailable()) return gw.sendText(event.chatId, 'Speech synthesis is not available right now, so spoken replies cannot be turned on. Try again later.');+ await setLinkVoiceReplies(link.id, link.user_id, arg === 'on');+ return gw.sendText(event.chatId, arg === 'on'+ ? 'Spoken replies are on. Every answer arrives as text, then as a voice note.'+ : 'Spoken replies are off. Answers arrive as text only.');+}+export async function cmdBalance({ gw, event, link }) {const agent = await needAgent(link, gw, event.chatId);if (!agent) return;@@ -168,6 +192,7 @@ export const COMMAND_HANDLERS = {use: cmdUse,unlink: cmdUnlink,new: cmdNew,+ voice: cmdVoice,balance: cmdBalance,portfolio: cmdPortfolio,runs: cmdRuns,diff --git a/api/_lib/gateway/conversation.js b/api/_lib/gateway/conversation.jsindex 42a59eda4..74609693b 100644--- a/api/_lib/gateway/conversation.js+++ b/api/_lib/gateway/conversation.js@@ -4,30 +4,45 @@// over the agent's cross-channel thread (api/_lib/agent-thread.js), so a// conversation started on the web continues here and the other way round. While// the agent works, the chat shows a typing indicator and one compact status-// line ("working: get_quote") edited in place. Every trade or limit change the-// agent proposes becomes a preview with Approve and Cancel buttons; nothing in-// the reply text can execute one.+// line ("working: get_quote") edited in place, on platforms that can edit a+// message (SMS, Signal and email cannot, so they get neither). Every trade or+// limit change the agent proposes becomes a preview with Approve and Cancel+// buttons, or on a channel without buttons a single-use reply code+// (./reply-codes.js); nothing in the reply text can execute one. When the+// owner turned on voice replies for the chat, the reply is also spoken+// (./voice.js).import { runCopilotTurn } from '../copilot-engine.js';import { appendThreadMessage, threadHistoryForModel } from '../agent-thread.js';import { resolveChatAgent, listAccountAgents } from './agents.js';import { createPreview, setPreviewMessageRef, PREVIEW_TTL_MINUTES } from './store.js';import { chunkText, describeProposal, plainText, MAX_TEXT, PLATFORM_LABEL, appOrigin } from './format.js';+import { maybeSpeakReply } from './voice.js';export const APPROVE = 'gw:ap:';export const CANCEL = 'gw:cx:';const TYPING_EVERY_MS = 4500;-function surfaceNote(platform) {+/** Whether a gateway renders buttons (true unless it says otherwise). */+export function hasButtons(gw) {+ return gw.buttons !== false;+}++function surfaceNote(platform, { buttons = true } = {}) {const where = PLATFORM_LABEL[platform] || platform;+ const approval = buttons+ ? 'Every propose_* card you surface reaches the owner as a preview message with Approve and Cancel buttons. Tell them to press Approve to execute it or Cancel to discard it, and that it expires in ten minutes. Never claim a trade happened.'+ : 'Every propose_* card you surface reaches the owner as a preview message ending with a six-digit code. Tell them to reply APPROVE and that code to execute it, or CANCEL and the code to discard it, and that it expires in ten minutes. Never claim a trade happened.';+ const brevity = platform === 'sms' ? ' This is SMS: answer in two or three short sentences.' : ' Keep it short.';return [- `SURFACE: you are replying in a ${where} chat with your owner. Write plain text: no tables, no headings, no code blocks. Keep it short.`,- 'Every propose_* card you surface reaches the owner as a preview message with Approve and Cancel buttons. Tell them to press Approve to execute it or Cancel to discard it, and that it expires in ten minutes. Never claim a trade happened.',+ `SURFACE: you are replying in a ${where} chat with your owner. Write plain text: no tables, no headings, no code blocks.${brevity}`,+ approval,].join('\n');}/** Keep the platform's typing indicator alive until stop() is called. */function keepTyping(gw, chatId) {+ if (typeof gw.typing !== 'function') return () => {};let stopped = false;const tick = () => { if (!stopped) Promise.resolve(gw.typing(chatId)).catch(() => {}); };tick();@@ -40,6 +55,9 @@ function keepTyping(gw, chatId) {* left behind as a compact record of what the agent looked at.*/function statusLine(gw, chatId) {+ // A channel that cannot edit would get one new message per tool call; on+ // SMS that is a bill, so those channels skip the status line entirely.+ if (gw.canEdit === false) return { working() {}, async finish() {} };let ref = null;let chain = Promise.resolve();const used = [];@@ -82,7 +100,7 @@ export function previewText(proposal) {async function sendPreviews({ gw, event, link, agent, proposals }) {for (const p of proposals) {const preview = await createPreview({ linkId: link.id, userId: link.user_id, agentId: agent.id, kind: p.kind, proposal: p });- const ref = await gw.sendChoice(event.chatId, previewText(p), choicesFor(preview, p));+ const ref = await gw.sendChoice(event.chatId, previewText(p), choicesFor(preview, p), { link });if (ref) await setPreviewMessageRef(preview.id, ref);}}@@ -117,7 +135,7 @@ export async function converse({ gw, event, link, text }) {agent,history,network: 'mainnet',- surfaceNote: surfaceNote(event.platform),+ surfaceNote: surfaceNote(event.platform, { buttons: hasButtons(gw) }),emit: (name, data) => { if (name === 'tool_start') status.working(data.name); },});} catch (e) {@@ -134,6 +152,7 @@ export async function converse({ gw, event, link, text }) {const reply = plainText(turn.reply) || (turn.proposals.length ? 'Here is what I prepared:' : 'I have nothing to add to that.');await sendReply(gw, event.chatId, reply);+ await maybeSpeakReply({ gw, chatId: event.chatId, link, text: reply, agentId: agent.id });if (turn.proposals.length) await sendPreviews({ gw, event, link, agent, proposals: turn.proposals });await appendThreadMessage({diff --git a/api/_lib/gateway/media.js b/api/_lib/gateway/media.jsindex 69fb4b1ea..305e7b52e 100644--- a/api/_lib/gateway/media.js+++ b/api/_lib/gateway/media.js@@ -1,10 +1,10 @@// Voice notes and photos sent to a chat gateway become text the agent can read.-// Voice runs through the same NVIDIA Riva speech path as POST /api/asr; photos-// run through the same vision chain as POST /api/vision.+// Voice runs through the gateway audio path (./audio.js: the NVIDIA Riva lane+// behind POST /api/asr, Gemini on Vertex as the failover, any container+// transcoded first); photos run through the same vision chain as POST /api/vision.-import { transcribeNvidiaAsr, nvidiaAsrConfigured } from '../asr-nvidia.js';import { describeImage, visionConfigured } from '../vision.js';-import { toSpeechInput, AudioError } from './audio.js';+import { transcribeAudio, speechConfigured } from './audio.js';export const MAX_VOICE_BYTES = 8 * 1024 * 1024;export const MAX_IMAGE_BYTES = 8 * 1024 * 1024;@@ -16,25 +16,23 @@ export class MediaError extends Error {}}+const UNREADABLE = new Set(['transcode_failed', 'transcode_timeout']);+/**- * Transcribe a chat voice note. Ogg/Opus (Telegram, Discord, WhatsApp) and FLAC- * go to Riva as-is; AAC, MP4 and WebM (Signal, iOS, Slack clips) are converted- * to 16 kHz PCM first (./audio.js).+ * Transcribe a chat voice note in whatever container the platform recorded it.* @returns {Promise<string>}*/export async function transcribeVoice({ buffer, mimeType = 'audio/ogg', language = 'en-US' }) {- if (!nvidiaAsrConfigured()) throw new MediaError('asr_unavailable', 'Voice notes are not available right now. Type your message instead.');+ if (!speechConfigured()) throw new MediaError('asr_unavailable', 'Voice notes are not available right now. Type your message instead.');if (!buffer?.length) throw new MediaError('empty_audio', 'That voice note was empty.');if (buffer.length > MAX_VOICE_BYTES) throw new MediaError('audio_too_large', 'That voice note is too long. Keep it under about five minutes.');- let input;+ let text;try {- input = await toSpeechInput({ buffer, mimeType });+ text = await transcribeAudio({ buffer, mimeType, language });} catch (e) {- if (e instanceof AudioError) throw new MediaError('audio_unreadable', 'I could not read that voice note. Try recording it again, or type it.');+ if (UNREADABLE.has(e?.code)) throw new MediaError('audio_unreadable', 'I could not read that voice note. Try recording it again, or type it.');throw e;}- const out = await transcribeNvidiaAsr({ audio: input.audio, encoding: input.encoding, sampleRateHz: input.sampleRateHz, language });- const text = String(out?.text || '').trim();if (!text) throw new MediaError('no_speech', 'I could not make out any words in that voice note.');return text;}diff --git a/api/_lib/gateway/voice.js b/api/_lib/gateway/voice.jsnew file mode 100644index 000000000..a1b4d889a--- /dev/null+++ b/api/_lib/gateway/voice.js@@ -0,0 +1,32 @@+// Spoken replies for chats with "voice replies" turned on (/voice on in the+// chat, or the toggle on /settings/connections). After the text reply lands,+// the same reply is rendered as a voice note (./audio.js synthesizeReply) and+// sent with the adapter's sendVoice. A channel whose adapter has no sendVoice+// (SMS) never offers the setting. A speech failure never fails the turn: the+// owner already has the text.++import { synthesizeReply, ttsLanes } from './audio.js';++/** Whether any TTS lane can speak right now. */+export function voiceRepliesAvailable() {+ return ttsLanes().length > 0;+}++/**+ * Speak `text` into the chat when the link asked for it and the platform can+ * play audio. Returns what happened, for logs and tests.+ * @returns {Promise<'sent'|'off'|'unsupported'|'empty'|'failed'>}+ */+export async function maybeSpeakReply({ gw, chatId, link, text, agentId = null }) {+ if (!link?.voice_replies) return 'off';+ if (typeof gw.sendVoice !== 'function') return 'unsupported';+ try {+ const clip = await synthesizeReply(text, { agentId: agentId || link.default_agent_id || null });+ if (!clip) return 'empty';+ await gw.sendVoice(chatId, clip);+ return 'sent';+ } catch (e) {+ console.warn('[gateway] spoken reply failed', e?.code || e?.message);+ return 'failed';+ }+}diff --git a/api/_lib/gateway/webhooks.js b/api/_lib/gateway/webhooks.jsindex fc8782f8d..0f3c573b8 100644--- a/api/_lib/gateway/webhooks.js+++ b/api/_lib/gateway/webhooks.js@@ -1,9 +1,9 @@// Verification and normalisation for the chat gateway webhooks. The receivers-// (api/gateway/telegram.js, api/gateway/discord.js) verify the platform's proof,+// (api/gateway/{telegram,discord,slack,whatsapp,sms}.js) verify the platform's proof,// queue the delivery in gateway_inbox and answer at once; the worker does the// slow part. Kept free of HTTP so the tests can drive every branch.-import { createHash, createPublicKey, timingSafeEqual, verify } from 'node:crypto';+import { createHash, createHmac, createPublicKey, timingSafeEqual, verify } from 'node:crypto';// ── Telegram ─────────────────────────────────────────────────────────────────@@ -89,3 +89,132 @@ export function discordInboxKeys(interaction) {if (interaction.type !== DISCORD_INTERACTION.APPLICATION_COMMAND && interaction.type !== DISCORD_INTERACTION.MESSAGE_COMPONENT) return null;return { dedupeKey: String(interaction.id), chatKey: `discord:${interaction.channel_id}` };}++// ── Slack ────────────────────────────────────────────────────────────────────++const SLACK_MAX_SKEW_S = 300;++/**+ * Verify Slack's v0 request signature: HMAC-SHA256 of `v0:<timestamp>:<raw body>`+ * keyed with the app's signing secret, sent as `v0=<hex>` in X-Slack-Signature.+ * A timestamp more than five minutes off is refused so a capture cannot be replayed.+ */+export function verifySlackSignature({ rawBody, signature, timestamp, signingSecret, now = Date.now() }) {+ if (!signingSecret || !rawBody) return false;+ if (!/^\d{1,12}$/.test(String(timestamp || ''))) return false;+ if (Math.abs(now / 1000 - Number(timestamp)) > SLACK_MAX_SKEW_S) return false;+ const body = Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(String(rawBody));+ const expected = `v0=${createHmac('sha256', signingSecret).update(`v0:${timestamp}:`).update(body).digest('hex')}`;+ return safeEqual(signature, expected);+}++/**+ * Classify a verified Slack delivery. The one endpoint receives three shapes:+ * Events API JSON, slash commands (form fields) and interactivity (a form field+ * `payload` holding JSON). Returns what the receiver should do with it.+ * @param {{ contentType:string, rawBody:Buffer|string }} req+ * @returns {{ type:'challenge', challenge:string }+ * | { type:'enqueue', dedupeKey:string, chatKey:string, payload:object }+ * | { type:'ignore' }}+ */+export function classifySlackDelivery({ contentType, rawBody }) {+ const text = Buffer.isBuffer(rawBody) ? rawBody.toString('utf8') : String(rawBody || '');+ if (/application\/json/i.test(contentType || '')) {+ let body;+ try { body = JSON.parse(text); } catch { return { type: 'ignore' }; }+ if (body?.type === 'url_verification' && typeof body.challenge === 'string') return { type: 'challenge', challenge: body.challenge };+ if (body?.type !== 'event_callback' || !body.event) return { type: 'ignore' };+ const ev = body.event;+ // The bot's own posts, edits and joins come back as events too.+ if (ev.bot_id || ev.subtype) return { type: 'ignore' };+ const direct = ev.type === 'message' && ev.channel_type === 'im';+ if (!direct && ev.type !== 'app_mention') return { type: 'ignore' };+ if (!ev.channel || !ev.user) return { type: 'ignore' };+ return { type: 'enqueue', dedupeKey: `event:${body.event_id || ev.client_msg_id || ev.ts}`, chatKey: `slack:${ev.channel}`, payload: { kind: 'slack_event', team_id: body.team_id, event: ev } };+ }+ const form = Object.fromEntries(new URLSearchParams(text));+ if (form.payload) {+ let p;+ try { p = JSON.parse(form.payload); } catch { return { type: 'ignore' }; }+ if (p?.type !== 'block_actions' || !Array.isArray(p.actions) || !p.actions.length) return { type: 'ignore' };+ const channel = p.channel?.id || p.container?.channel_id;+ if (!channel) return { type: 'ignore' };+ const a = p.actions[0];+ return { type: 'enqueue', dedupeKey: `action:${p.trigger_id || `${a.action_id}:${a.action_ts}`}`, chatKey: `slack:${channel}`, payload: { kind: 'slack_action', action: p } };+ }+ if (form.command && form.channel_id && form.user_id) {+ return { type: 'enqueue', dedupeKey: `command:${form.trigger_id || `${form.channel_id}:${form.user_id}:${Date.now()}`}`, chatKey: `slack:${form.channel_id}`, payload: { kind: 'slack_command', command: form } };+ }+ return { type: 'ignore' };+}++// ── WhatsApp (Cloud API) ─────────────────────────────────────────────────────++/**+ * Verify Meta's X-Hub-Signature-256: `sha256=` + HMAC-SHA256 of the raw body+ * keyed with the app secret.+ */+export function verifyWhatsAppSignature({ rawBody, signature, appSecret }) {+ if (!appSecret || !rawBody) return false;+ const body = Buffer.isBuffer(rawBody) ? rawBody : Buffer.from(String(rawBody));+ const expected = `sha256=${createHmac('sha256', appSecret).update(body).digest('hex')}`;+ return safeEqual(String(signature || '').toLowerCase(), expected);+}++/**+ * Answer Meta's subscription handshake (GET ?hub.mode=subscribe&hub.verify_token=+ * &hub.challenge=). Returns the challenge to echo, or null to refuse.+ */+export function whatsappChallenge(query, verifyToken) {+ if (!verifyToken) return null;+ if (query?.['hub.mode'] !== 'subscribe') return null;+ if (!safeEqual(query?.['hub.verify_token'], verifyToken)) return null;+ const challenge = String(query?.['hub.challenge'] || '');+ return /^[\w-]{1,128}$/.test(challenge) ? challenge : null;+}++/**+ * One Cloud API webhook can batch several messages (and status receipts, which+ * the gateway ignores). Each message becomes its own inbox row.+ * @returns {Array<{ dedupeKey:string, chatKey:string, payload:object }>}+ */+export function whatsappInboxEntries(body) {+ const out = [];+ if (body?.object !== 'whatsapp_business_account') return out;+ for (const entry of body.entry || []) {+ for (const change of entry.changes || []) {+ const value = change?.value;+ if (change?.field !== 'messages' || !value) continue;+ for (const message of value.messages || []) {+ if (!message?.id || !message.from) continue;+ const contact = (value.contacts || []).find((c) => c.wa_id === message.from) || null;+ out.push({+ dedupeKey: String(message.id),+ chatKey: `whatsapp:${message.from}`,+ payload: { kind: 'whatsapp_message', phone_number_id: value.metadata?.phone_number_id || null, contact, message },+ });+ }+ }+ }+ return out;+}++// ── SMS (Twilio) ─────────────────────────────────────────────────────────────++/**+ * Verify X-Twilio-Signature: base64 HMAC-SHA1, keyed with the auth token, of the+ * exact public URL Twilio posted to followed by every form field as key+value,+ * keys sorted.+ */+export function verifyTwilioSignature({ url, params, signature, authToken }) {+ if (!authToken || !url || !signature) return false;+ const data = Object.keys(params || {}).sort().reduce((s, k) => s + k + String(params[k] ?? ''), String(url));+ const expected = createHmac('sha1', authToken).update(Buffer.from(data, 'utf8')).digest('base64');+ return safeEqual(signature, expected);+}++/** @returns {{ dedupeKey:string, chatKey:string } | null} */+export function smsInboxKeys(params) {+ if (!params?.MessageSid || !params.From) return null;+ return { dedupeKey: String(params.MessageSid), chatKey: `sms:${params.From}` };+}