Skip to content

armgit / bygga

Public Imported from GitHub
0

IKEA assembly, step by step: animated, manual-style 3D guides. Scan any paper manual with Together AI.

TypeScript

Phonetic spellings for product names in the voice

Youssef7335de3be41f
diff --git a/package.json b/package.json
index 23f8b4a..9f7e090 100644
--- a/package.json
+++ b/package.json
@@ -7,7 +7,8 @@
"build": "next build",
"start": "next start",
"lint": "eslint",
- "voice": "node --experimental-transform-types --no-warnings --import ./scripts/voice/register.mjs scripts/voice/record.ts"
+ "voice": "node --experimental-transform-types --no-warnings --import ./scripts/voice/register.mjs scripts/voice/record.ts",
+ "voice:names": "node --experimental-transform-types --no-warnings --import ./scripts/voice/register.mjs scripts/voice/names.ts"
},
"dependencies": {
"@vercel/analytics": "^1.6.1",
diff --git a/scripts/voice/names.ts b/scripts/voice/names.ts
new file mode 100644
index 0000000..c67109c
--- /dev/null
+++ b/scripts/voice/names.ts
@@ -0,0 +1,107 @@
+/**
+ * How the voice says product names. Kokoro reads "Malm" like an English word,
+ * so each name gets a phonetic spelling (Kokoro's [word](/ipa/) markup) the way
+ * an English speaker in an IKEA store would say it.
+ *
+ * npm run voice:names spell every catalog name that has no spelling yet
+ * npm run voice:names -- --check say a sample both ways and score what Whisper hears
+ *
+ * Spellings live in src/lib/voice/names.ts and are applied only when a line is
+ * rendered (src/lib/voice/tts.ts), so clip names and the on-screen text don't change.
+ */
+import { existsSync, readFileSync, writeFileSync } from "node:fs";
+import path from "node:path";
+import { PRODUCTS } from "@/lib/catalog";
+import { spokenName } from "@/lib/voice/script";
+import { forTTS, speak } from "@/lib/voice/tts";
+
+const FILE = path.resolve(import.meta.dirname, "../../src/lib/voice/names.ts");
+const key = process.env.TOGETHER_API_KEY?.trim();
+if (!key) throw new Error("TOGETHER_API_KEY is not set");
+const args = process.argv.slice(2);
+const names: Record<string, string> = existsSync(FILE) ? JSON.parse(readFileSync(FILE, "utf8").replace(/^[^{]*/, "").replace(/;\s*$/, "")) : {};
+const save = () =>
+ writeFileSync(FILE, `/** How the voice says product names, as Kokoro phonemes. Written by scripts/voice/names.ts. */\nexport const NAMES: Record<string, string> = ${JSON.stringify(Object.fromEntries(Object.entries(names).sort()), null, 1)};\n`);
+
+// Kokoro's American English phonemes (misaki), so a spelling never contains a sound it can't make
+const OK = /^[ˈˌabdefhijklmnpstuvwzæðŋɑɐɔəɛɜɪʊʌɡɹʃʒθʤʧᵊᵻɚAIOWYː ]+$/;
+
+const PROMPT = `You write how IKEA product names are said out loud in an American IKEA store: the familiar, slightly Swedish pronunciation an English speaker would use (MALM is "mahlm", POÄNG is "poh-ENG", IDÅSEN is "ee-DOH-sen", KALLAX is "KAH-lahks", HEMNES is "HEM-ness", BILLY is "BILL-ee").
+Write each in IPA using ONLY these symbols: ˈ ˌ a b d e f h i j k l m n p s t u v w z æ ð ŋ ɑ ɐ ɔ ə ɛ ɜ ɪ ʊ ʌ ɡ ɹ ʃ ʒ θ ʤ ʧ ɚ ː and the diphthongs A (as in day), I (as in my), O (as in go), W (as in now), Y (as in boy). Put ˈ before the stressed syllable.
+Reply with JSON only: {"NAME": "ipa", ...} for every name given.`;
+
+async function ask(list: string[]) {
+ const r = await fetch("https://api.together.xyz/v1/chat/completions", {
+ method: "POST",
+ headers: { Authorization: `Bearer ${key}`, "Content-Type": "application/json" },
+ body: JSON.stringify({
+ model: "openai/gpt-oss-120b",
+ reasoning_effort: "medium",
+ response_format: { type: "json_object" },
+ max_tokens: 6000,
+ temperature: 0.2,
+ messages: [
+ { role: "system", content: PROMPT },
+ { role: "user", content: list.join("\n") },
+ ],
+ }),
+ });
+ if (!r.ok) throw new Error(`chat ${r.status}`);
+ const text: string = (await r.json()).choices?.[0]?.message?.content ?? "{}";
+ return JSON.parse(text.slice(text.indexOf("{"), text.lastIndexOf("}") + 1)) as Record<string, string>;
+}
+
+async function transcribe(buf: ArrayBuffer) {
+ const form = new FormData();
+ form.append("file", new Blob([buf], { type: "audio/mpeg" }), "clip.mp3");
+ form.append("model", "openai/whisper-large-v3");
+ form.append("language", "en");
+ const r = await fetch("https://api.together.ai/v1/audio/transcriptions", { method: "POST", headers: { Authorization: `Bearer ${key}` }, body: form });
+ return String((await r.json()).text ?? "");
+}
+
+const fold = (s: string) => s.toLowerCase().normalize("NFD").replace(/[̀-ͯ]/g, "").replace(/[^a-z]/g, "");
+function sim(a: string, b: string) {
+ const d = Array.from({ length: a.length + 1 }, (_, i) => [i, ...Array(b.length).fill(0)]);
+ for (let j = 1; j <= b.length; j++) d[0][j] = j;
+ for (let i = 1; i <= a.length; i++) for (let j = 1; j <= b.length; j++) d[i][j] = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + (a[i - 1] === b[j - 1] ? 0 : 1));
+ return 1 - d[a.length][b.length] / Math.max(a.length, b.length, 1);
+}
+
+async function main() {
+ const all = [...new Set(PRODUCTS.map((p) => spokenName(p.name)))];
+ if (args.includes("--check")) {
+ const sample = all.filter((n) => names[n]).slice(0, Number(args[args.indexOf("--check") + 1]) || 30);
+ let plain = 0, spelled = 0;
+ for (const n of sample) {
+ const line = `Your ${n} is built.`;
+ const [a, b] = await Promise.all([speak(line, key!), speak(forTTS(line), key!)]);
+ const [ha, hb] = await Promise.all([transcribe(a), transcribe(b)]);
+ const word = (h: string) => fold(h.replace(/^\W*your\s+/i, "").replace(/\s+is built.*$/i, ""));
+ const sa = sim(fold(n), word(ha)), sb = sim(fold(n), word(hb));
+ plain += sa;
+ spelled += sb;
+ console.log(`${n.padEnd(12)} plain "${word(ha)}" ${sa.toFixed(2)} spelled "${word(hb)}" ${sb.toFixed(2)} ${names[n]}`);
+ }
+ console.log(`average likeness to the written name: plain ${(plain / sample.length).toFixed(2)}, spelled ${(spelled / sample.length).toFixed(2)}`);
+ return;
+ }
+ const todo = all.filter((n) => !names[n]);
+ console.log(`${all.length} names, ${todo.length} to spell`);
+ for (let i = 0; i < todo.length; i += 60) {
+ const got = await ask(todo.slice(i, i + 60).map((n) => n.toUpperCase()));
+ for (const [k, v] of Object.entries(got)) {
+ const n = todo.find((x) => x.toUpperCase() === k.toUpperCase());
+ const ipa = String(v).trim().replace(/^\/|\/$/g, "");
+ if (n && OK.test(ipa)) names[n] = ipa;
+ else if (n) console.log(` skipped ${n}: ${ipa}`);
+ }
+ save();
+ console.log(`${Math.min(i + 60, todo.length)}/${todo.length}`);
+ }
+}
+
+main().catch((e) => {
+ console.error(e);
+ process.exit(1);
+});
diff --git a/scripts/voice/record.ts b/scripts/voice/record.ts
index 36d396a..ae7f19f 100644
--- a/scripts/voice/record.ts
+++ b/scripts/voice/record.ts
@@ -22,7 +22,7 @@ import { specForKind } from "@/lib/search";
import { SAMPLE_SCAN } from "@/lib/scan/sample";
import type { Plan } from "@/lib/types";
import { LINES, VOICE_DIR, builtLine, clipId, numWords, stepLines, whereLine } from "@/lib/voice/script";
-import { TTS, speak } from "@/lib/voice/tts";
+import { TTS, forTTS, speak } from "@/lib/voice/tts";
const ROOT = path.resolve(import.meta.dirname, "../..");
const OUT = path.join(ROOT, "public", VOICE_DIR);
@@ -68,7 +68,7 @@ const env = (k: string) => {
};
async function render(text: string): Promise<Buffer> {
- return Buffer.from(await speak(DIRECTION + text, env("TOGETHER_API_KEY")));
+ return Buffer.from(await speak(DIRECTION + forTTS(text), env("TOGETHER_API_KEY")));
}
/** Trim silence, fade the edges, match loudness (two passes), mono 64 kbps mp3. */
diff --git a/src/app/api/voice/route.ts b/src/app/api/voice/route.ts
index 1db970e..0e08cb8 100644
--- a/src/app/api/voice/route.ts
+++ b/src/app/api/voice/route.ts
@@ -1,6 +1,6 @@
import { connection } from "next/server";
import { hasKey, ipOf, rateLimited } from "@/lib/together";
-import { speak } from "@/lib/voice/tts";
+import { forTTS, speak } from "@/lib/voice/tts";
/**
* Records a line live in the same voice as the pre-recorded clips, for text
@@ -17,7 +17,7 @@ export async function GET(req: Request) {
if (!text || text.length > 400) return Response.json({ error: "bad text" }, { status: 400 });
if (rateLimited(ipOf(req) + ":voice", 90)) return Response.json({ error: "slow down" }, { status: 429 });
try {
- const audio = await speak(text, process.env.TOGETHER_API_KEY!.trim());
+ const audio = await speak(forTTS(text), process.env.TOGETHER_API_KEY!.trim());
return new Response(audio, {
headers: { "Content-Type": "audio/mpeg", "Cache-Control": "public, max-age=31536000, immutable" },
});
diff --git a/src/lib/voice/names.ts b/src/lib/voice/names.ts
new file mode 100644
index 0000000..f6b6888
--- /dev/null
+++ b/src/lib/voice/names.ts
@@ -0,0 +1,2 @@
+/** How the voice says product names, as Kokoro phonemes. Written by scripts/voice/names.ts. */
+export const NAMES: Record<string, string> = {};
diff --git a/src/lib/voice/tts.ts b/src/lib/voice/tts.ts
index ccf4f75..88874dd 100644
--- a/src/lib/voice/tts.ts
+++ b/src/lib/voice/tts.ts
@@ -1,3 +1,5 @@
+import { NAMES as SPELLED } from "./names";
+
/**
* The hands-free voice, on Together AI: one model and voice for the
* pre-recorded clips (scripts/voice/record.ts) and for lines recorded live
@@ -11,6 +13,15 @@ export const TTS = {
usdPer1K: 0.004,
};
+const NAMES = Object.entries(SPELLED);
+
+/** The line as the voice should read it: product names get their phonetic spelling (scripts/voice/names.ts). */
+export function forTTS(text: string) {
+ let t = text;
+ for (const [name, ipa] of NAMES) if (t.includes(name)) t = t.replace(new RegExp(`(?<!\\p{L})${name}(?!\\p{L})`, "gu"), `[${name}](/${ipa}/)`);
+ return t;
+}
+
/** Renders one line to mp3. The key is the server's, never the visitor's. */
export async function speak(text: string, key: string) {
const r = await fetch("https://api.together.ai/v1/audio/speech", {