Skip to content

🎬 火宝短剧 - 基于AI的一站式短剧生成平台 《一句话生成完整短剧,从剧本到成片全自动化》 Huobao Drama - An AI-Powered End-to-End Short Drama Generator "One Sentence to Complete Drama: Fully Automated from Script to Final Video"

TypeScript

feat: add character voice uploads and inline video prompt references

Connor70c14ea3ea37
diff --git a/backend/src/agents/index.ts b/backend/src/agents/index.ts
index 7c21fe9..27dbcfb 100644
--- a/backend/src/agents/index.ts
+++ b/backend/src/agents/index.ts
@@ -18,6 +18,7 @@ import { loadAgentSkills, skillWorkspaces } from './skills.js'
import { loadAgentPromptFile, loadBasePromptFile } from './prompts.js'
import { buildLanguageDirective } from './language.js'
import { getContentLanguageFromRC } from './context.js'
+import { getVoiceReferenceInstructions } from './voice-reference-instructions.js'
// Default prompts (used when workspace/prompts/<type>.md 文件缺失时兜底)
export const DEFAULT_PROMPTS: Record<string, { name: string; instructions: string }> = {
@@ -356,7 +357,8 @@ function buildInstructions(type: string) {
const baseInstructions = promptFile?.instructions || defaults.instructions
const skillInstructions = await loadAgentSkills(type, lang)
const languageDirective = buildLanguageDirective(lang)
- return [baseInstructions, skillInstructions, languageDirective]
+ const voiceInstructions = getVoiceReferenceInstructions(type, lang)
+ return [baseInstructions, skillInstructions, voiceInstructions, languageDirective]
.filter(Boolean)
.join('\n\n')
}
diff --git a/backend/src/agents/language.ts b/backend/src/agents/language.ts
index c89d049..9720a99 100644
--- a/backend/src/agents/language.ts
+++ b/backend/src/agents/language.ts
@@ -26,6 +26,7 @@ export function buildLanguageDirective(lang?: string | null): string {
'',
'Exceptions:',
'- When referencing EXISTING assets with @mentions (e.g. in video prompts), the name after @ must EXACTLY match the asset name as it appears in the provided asset lists — do NOT translate or rewrite existing asset names.',
+ '- Character voice references (e.g. #Alex) are literal tokens: copy the complete voice_reference value exactly, including the existing name, regardless of output language. Do not add a suffix.',
'- The visual style prefix of image prompts is injected automatically by the system (in English). Do not translate, rewrite, or duplicate it.',
].join('\n')
}
diff --git a/backend/src/agents/tools/storyboard-tools.ts b/backend/src/agents/tools/storyboard-tools.ts
index d1c4271..d0ed29a 100644
--- a/backend/src/agents/tools/storyboard-tools.ts
+++ b/backend/src/agents/tools/storyboard-tools.ts
@@ -6,10 +6,19 @@ import { createTool } from '@mastra/core/tools'
import type { ToolExecutionContext } from '@mastra/core/tools'
import { z } from 'zod'
import { db, getInsertId, schema } from '../../db/index.js'
-import { eq } from 'drizzle-orm'
+import { and, eq, isNull } from 'drizzle-orm'
import { now } from '../../utils/response.js'
import { logTaskProgress, logTaskSuccess } from '../../utils/task-logger.js'
import { getDramaId, getEpisodeId } from '../context.js'
+import { resolveVoiceMentions } from '../../services/character-voice-references.js'
+
+async function validateVideoPromptVoices(prompt: string | undefined, dramaId: number, characterIds: number[] = []) {
+ if (!prompt?.includes('#')) return
+ const characters = await db.select({ id: schema.characters.id, name: schema.characters.name, voiceAudioUrl: schema.characters.voiceAudioUrl }).from(schema.characters)
+ .where(and(eq(schema.characters.dramaId, dramaId), isNull(schema.characters.deletedAt)))
+ // Validate the model's references, but persist the original editable #tokens.
+ resolveVoiceMentions(prompt, characters.map(c => ({ ...c, bound: characterIds.includes(c.id) })))
+}
async function syncStoryboardCharacters(storyboardId: number, characterIds: number[]) {
await db.delete(schema.storyboardCharacters)
@@ -150,6 +159,8 @@ const readStoryboardContext = createTool({
styling: c.styling || '',
image_url: c.imageUrl || '',
reference_images: c.referenceImages || '',
+ has_voice: Boolean(c.voiceAudioUrl?.trim()),
+ voice_reference: c.voiceAudioUrl?.trim() ? `#${c.name}` : null,
}))
const scenes = scns
@@ -189,6 +200,9 @@ const readStoryboardContext = createTool({
title: sb.title || '',
scene_id: sb.sceneId,
character_ids: links.map(link => link.characterId),
+ available_voice_references: characters
+ .filter(c => c.has_voice && links.some(link => link.characterId === c.id))
+ .map(c => ({ character_id: c.id, name: c.name, voice_reference: c.voice_reference })),
prop_ids: sbPropLinks.map(link => link.propId),
shot_type: sb.shotType || '',
duration: sb.duration || 0,
@@ -264,6 +278,10 @@ const saveStoryboards = createTool({
count: storyboards.length,
shotNumbers: storyboards.map(sb => sb.shot_number).join(','),
})
+ // Check all voice references before replace_existing can remove the previous storyboards.
+ for (const sb of storyboards) {
+ await validateVideoPromptVoices(sb.video_prompt, dramaId, sb.character_ids)
+ }
if (replace_existing === true) {
const existingStoryboardRows = await db.select().from(schema.storyboards)
.where(eq(schema.storyboards.episodeId, episodeId))
@@ -369,7 +387,7 @@ const updateStoryboard = createTool({
if ('error' in ids) return ids
const { episodeId, dramaId } = ids
const [storyboard] = await db.select().from(schema.storyboards).where(eq(schema.storyboards.id, storyboard_id))
- if (!storyboard) return { error: `Storyboard ${storyboard_id} not found` }
+ if (!storyboard || storyboard.episodeId !== episodeId || storyboard.deletedAt) return { error: `Storyboard ${storyboard_id} not found in this episode` }
// 过滤模型回传的垃圾值:视频提示词 Agent 常把整行字段回传,
// 拿不准的字符串字段写成 "null"/"undefined",直接覆盖会毁掉已有内容
@@ -398,6 +416,7 @@ const updateStoryboard = createTool({
.where(eq(schema.storyboardProps.storyboardId, storyboard_id)))
.map(link => link.propId)
+ await validateVideoPromptVoices(fields.video_prompt, dramaId, currentCharacterIds)
await validateStoryboardBindings(
episodeId,
dramaId,
diff --git a/backend/src/agents/voice-reference-instructions.ts b/backend/src/agents/voice-reference-instructions.ts
new file mode 100644
index 0000000..54be122
--- /dev/null
+++ b/backend/src/agents/voice-reference-instructions.ts
@@ -0,0 +1,46 @@
+/** Shared by first-time breakdown and single/batch video prompt generation.
+ * Inject at runtime so existing/custom desktop prompt files also see the voice contract.
+ */
+const instructions: Record<string, string> = {
+ zh: `## 视频提示词中的角色音色引用
+- read_storyboard_context 的角色提供 has_voice 与 voice_reference;已有分镜提供 available_voice_references。只有 has_voice 为 true 的角色才有可用音色。
+- 为某个分镜生成或重写 video_prompt 时,只使用该分镜 available_voice_references 中的音色;首次拆分时,只使用本段 character_ids 对应角色的 voice_reference。
+- 先从 description 的台词确定说话角色。画面动作使用 @角色名;有台词且有可用音色时,在对应对白的说话人位置直接使用 #角色名,紧接语气、发声动作与台词。例如:0-3秒:@林小雨抬头,#林小雨轻声说:「早上好。」
+- 音色引用必须与对白写在同一个时间段内,不新增独立的「音色:」信息头。每次对白都明确标出自己的 #说话人;多人对话逐句对应,不能把所有音色集中列在开头。
+- 直接复制 voice_reference 的完整值(如 #林小雨),禁止改名、缩写、翻译或追加“音色”后缀。@角色名引用形象,#角色名引用音色,两者分别放在画面动作与发声位置。
+- 没有上传音色的角色、未绑定角色、已移除音色和不说话的角色,不添加 # 引用。旁白只有明确对应某个已绑定、已有音色的角色时才可引用,不借用其他角色的音色。
+- 音频仅用于参考音色,台词内容与说话时机遵循 description;不要为了使用音色新增台词、人物或镜头。
+- 保存前检查 @图片引用、#音色引用和说话角色的对应关系。video_prompt 中保留原始 #角色名 标签,不写音频 URL、Base64、音频1 等上游编号;编号由视频生成接口自动映射。`,
+ en: `## Character voice references in video_prompt
+- read_storyboard_context provides has_voice and voice_reference for each character, plus available_voice_references for each existing storyboard.
+- For an existing shot, use only its available_voice_references. During initial breakdown, use only voice_reference values belonging to this segment's character_ids.
+- Identify speakers from description. Use @CharacterName for visual actions and #CharacterName directly as the speaker before delivery and dialogue, e.g. "0-3s: @Alex looks up; #Alex softly says: 'Good morning.'"
+- Keep voice references inline with each utterance in its time segment. Do not add a separate voice header or collect all voices at the start. Mark the correct #speaker for every utterance, including repeated speakers and multi-person dialogue.
+- Copy voice_reference exactly (e.g. #Alex). Do not translate, abbreviate or add a suffix. @CharacterName is the image; #CharacterName is the voice.
+- Do not add # references for unbound characters, missing/removed uploads, or silent characters. Narration may use a voice only when its speaker is explicitly a bound character with that voice; never borrow another character's voice.
+- Reference audio supplies timbre only: preserve the original dialogue and timing from description, and do not add dialogue, characters or shots just to use a voice.
+- Before saving, check speaker/image/voice bindings. Save the original #CharacterName tokens, never audio URLs, Base64 or provider audio numbers; video generation resolves the indices.`,
+ ja: `## video_prompt のキャラクター音声参照
+- read_storyboard_context は各キャラクターの has_voice と voice_reference、既存ショットの available_voice_references を返す。
+- 既存ショットではその available_voice_references のみ使用する。初回分割では、その段落の character_ids に含まれるキャラクターの voice_reference のみ使用する。
+- description のセリフから話者を確認する。画面の動作は @名前、音声がある話者の発声は #名前 を直接使う。例:「0-3秒:@太郎が顔を上げ、#太郎が優しく『おはよう』と言う。」
+- 音声参照は各時間帯のセリフ内に記載する。独立した音声ヘッダーを追加せず、冒頭に音声をまとめない。発話ごとに正しい #話者 を指定する。
+- voice_reference(例:#太郎)を完全一致でコピーし、翻訳・省略・接尾辞の追加をしない。@名前 は画像、#名前 は音声の参照。
+- 未紐付け・未アップロード・削除済み音声・無言の人物には # 参照を付けない。ナレーションは話者が音声付きの紐付け済みキャラクターと明示されている場合のみ参照し、別人の声を流用しない。
+- 音声は声質の参照のみ。description のセリフと発話タイミングを守り、音声を使うためのセリフ・人物・カットを追加しない。
+- 保存前に話者・画像・音声の対応を確認する。video_prompt には元の #名前 を保存し、音声URL、Base64、音声番号を書かない。番号への変換は動画生成側が行う。`,
+ ko: `## video_prompt의 캐릭터 음색 참조
+- read_storyboard_context는 캐릭터별 has_voice와 voice_reference, 기존 샷별 available_voice_references를 제공한다.
+- 기존 샷은 해당 available_voice_references만 사용한다. 최초 분할은 해당 구간 character_ids에 속하는 캐릭터의 voice_reference만 사용한다.
+- description의 대사에서 화자를 확인한다. 화면 동작은 @이름, 음색이 있는 화자의 발화는 #이름을 직접 사용한다. 예: "0-3초: @민수가 고개를 들고, #민수가 부드럽게 '안녕'이라고 말한다."
+- 각 시간 구간의 대사 안에 음색 참조를 넣는다. 별도의 음색 헤더를 추가하거나 처음에 음색을 모아 쓰지 않는다. 대사마다 정확한 #화자를 표시한다.
+- voice_reference(예: #민수)를 그대로 복사한다. 번역, 축약, 접미사 추가를 하지 않는다. @이름은 이미지, #이름은 음색 참조다.
+- 미연결 캐릭터, 미업로드/삭제된 음색, 말하지 않는 캐릭터에는 # 참조를 추가하지 않는다. 내레이션은 화자가 음색이 있는 연결 캐릭터로 명시된 경우에만 참조하며 다른 캐릭터의 음색을 빌리지 않는다.
+- 오디오는 음색만 참고한다. description의 대사와 발화 시점을 유지하고 음색 사용을 위해 대사, 인물, 샷을 추가하지 않는다.
+- 저장 전 화자/이미지/음색 연결을 확인한다. video_prompt에는 원래 #이름 토큰을 저장한다. 오디오 URL, Base64, 공급자 오디오 번호는 쓰지 않는다. 번호 변환은 영상 생성 단계에서 수행한다.`,
+}
+
+export function getVoiceReferenceInstructions(agentType: string, language?: string | null): string {
+ if (agentType !== 'prompt_generator' && agentType !== 'storyboard_breaker') return ''
+ return instructions[language || 'zh'] || instructions.zh
+}
diff --git a/backend/src/db/schema.ts b/backend/src/db/schema.ts
index ec5ba67..a922426 100644
--- a/backend/src/db/schema.ts
+++ b/backend/src/db/schema.ts
@@ -54,6 +54,8 @@ export const characters = sqliteTable('characters', {
finalPrompt: text('final_prompt'),
personality: text('personality'),
imageUrl: text('image_url'),
+ voiceAudioUrl: text('voice_audio_url'),
+ voiceAudioName: text('voice_audio_name'),
referenceImages: text('reference_images'),
seedValue: text('seed_value'),
sortOrder: integer('sort_order'),
diff --git a/backend/src/db/sqlite-schema.ts b/backend/src/db/sqlite-schema.ts
index f2f25f1..fc26bc5 100644
--- a/backend/src/db/sqlite-schema.ts
+++ b/backend/src/db/sqlite-schema.ts
@@ -56,6 +56,8 @@ export const sqliteSchemaStatements = [
final_prompt TEXT,
personality TEXT,
image_url TEXT,
+ voice_audio_url TEXT,
+ voice_audio_name TEXT,
reference_images TEXT,
seed_value TEXT,
sort_order INTEGER,
@@ -383,6 +385,11 @@ export function initSqliteSchema(sqlite: Database.Database) {
for (const [name, type] of Object.entries({ config_id: 'INTEGER', config_base_url: 'TEXT', stage: 'TEXT', error_code: 'TEXT' })) {
if (!taskColumns.has(name)) sqlite.exec(`ALTER TABLE sys_task ADD COLUMN ${name} ${type}`)
}
+ // 角色音色由用户上传;旧库补充可空字段,不修改已有角色和图片。
+ const characterColumns = new Set((sqlite.prepare('PRAGMA table_info(characters)').all() as { name: string }[]).map(c => c.name))
+ for (const name of ['voice_audio_url', 'voice_audio_name']) {
+ if (!characterColumns.has(name)) sqlite.exec(`ALTER TABLE characters ADD COLUMN ${name} TEXT`)
+ }
const insertSeed = sqlite.prepare(SEED_SQL)
const upgradeSeed = sqlite.prepare(UPGRADE_SQL)
const removeSeed = sqlite.prepare(REMOVE_SQL)
diff --git a/backend/src/middleware/logger.ts b/backend/src/middleware/logger.ts
index 4909e57..412b644 100644
--- a/backend/src/middleware/logger.ts
+++ b/backend/src/middleware/logger.ts
@@ -32,7 +32,9 @@ export const requestLogger: MiddlewareHandler = async (c, next) => {
// 打印请求
const time = formatTime()
let bodyInfo = ''
- if (['POST', 'PUT', 'PATCH'].includes(method)) {
+ // 上传文件不克隆/读取请求体:避免记录音频内容,也让路由的大小限制先执行。
+ const isFileUpload = c.req.header('content-type')?.toLowerCase().startsWith('multipart/form-data')
+ if (!isFileUpload && ['POST', 'PUT', 'PATCH'].includes(method)) {
try {
const clone = c.req.raw.clone()
const text = await clone.text()
diff --git a/backend/src/routes/characters.ts b/backend/src/routes/characters.ts
index ddd0820..f920668 100644
--- a/backend/src/routes/characters.ts
+++ b/backend/src/routes/characters.ts
@@ -1,7 +1,11 @@
import { Hono } from 'hono'
-import { and, eq } from 'drizzle-orm'
+import { bodyLimit } from 'hono/body-limit'
+import { and, eq, isNull } from 'drizzle-orm'
+import fs from 'node:fs/promises'
import { db, getInsertId, schema } from '../db/index.js'
-import { success, created, badRequest, now } from '../utils/response.js'
+import { success, created, badRequest, notFound, now } from '../utils/response.js'
+import { AUDIO_MAX_BYTES, AudioUploadError, saveAudioUpload } from '../utils/audio-upload.js'
+import { getAbsolutePath } from '../utils/storage.js'
import { toSnakeCase } from '../utils/transform.js'
import { generateImage } from '../services/generation.js'
import { getDramaStylePrompt } from '../services/style-preset.js'
@@ -11,6 +15,47 @@ import { logTaskError, logTaskStart, logTaskSuccess } from '../utils/task-logger
const app = new Hono()
const CHARACTER_IMAGE_SIZE = '1920x1080'
+// 用户上传的角色音色样本独立于生图和 AI 提取,跨剧集复用同一个角色绑定。
+app.post('/:id/voice', bodyLimit({ maxSize: AUDIO_MAX_BYTES + 64 * 1024,
+ onError: c => badRequest(c, '音频文件大小不能超过 20MB'),
+}), async c => {
+ const id = Number(c.req.param('id'))
+ if (!Number.isSafeInteger(id) || id <= 0) return badRequest(c, '角色编号无效')
+ const condition = and(eq(schema.characters.id, id), isNull(schema.characters.deletedAt))
+ const char = db.select({ id: schema.characters.id }).from(schema.characters).where(condition).get()
+ if (!char) return notFound(c, '角色不存在')
+ let uploadedPath: string | undefined
+ try {
+ const body = await c.req.parseBody()
+ const audio = await saveAudioUpload(body.file)
+ uploadedPath = audio.path
+ const updated = db.update(schema.characters)
+ .set({ voiceAudioUrl: audio.path, voiceAudioName: audio.name, updatedAt: now() })
+ .where(condition).returning({ id: schema.characters.id }).get()
+ if (!updated) {
+ await fs.unlink(getAbsolutePath(audio.path))
+ return notFound(c, '角色不存在')
+ }
+ return success(c, { voice_audio_url: audio.path, voice_audio_name: audio.name })
+ } catch (error) {
+ if (uploadedPath) await fs.unlink(getAbsolutePath(uploadedPath)).catch(() => {})
+ if (error instanceof AudioUploadError) return badRequest(c, error.message)
+ throw error
+ }
+})
+
+// 只移除角色绑定,不删除可能仍被引用的媒体文件。
+app.delete('/:id/voice', async c => {
+ const id = Number(c.req.param('id'))
+ if (!Number.isSafeInteger(id) || id <= 0) return badRequest(c, '角色编号无效')
+ const updated = db.update(schema.characters)
+ .set({ voiceAudioUrl: null, voiceAudioName: null, updatedAt: now() })
+ .where(and(eq(schema.characters.id, id), isNull(schema.characters.deletedAt)))
+ .returning({ id: schema.characters.id }).get()
+ if (!updated) return notFound(c, '角色不存在')
+ return success(c, { voice_audio_url: null, voice_audio_name: null })
+})
+
// POST /characters — 手动新增角色(传入 episode_id 时关联到该集)
app.post('/', async (c) => {
const body = await c.req.json()
diff --git a/backend/src/routes/tasks.ts b/backend/src/routes/tasks.ts
index 2461742..2776f85 100644
--- a/backend/src/routes/tasks.ts
+++ b/backend/src/routes/tasks.ts
@@ -6,6 +6,7 @@ import { generateImage, generateVideo, recoverVideoTask } from '../services/gene
import { safeTaskError } from '../services/video-download.js'
import { getActiveConfig, getConfigById } from '../services/ai.js'
import { getDramaStylePrompt } from '../services/style-preset.js'
+import { resolveStoryboardVoiceReferences } from '../services/character-voice-references.js'
import { logTaskError, logTaskPayload, logTaskStart, logTaskSuccess } from '../utils/task-logger.js'
const app = new Hono()
@@ -166,6 +167,12 @@ app.post('/', async (c) => {
: await getActiveConfig('video')
const validationError = validateVideoRequest(videoBody, effectiveConfig?.provider)
if (validationError) return badRequest(c, validationError)
+ const voices = await resolveStoryboardVoiceReferences(String(videoBody.prompt || ''), Number(body.storyboard_id) || undefined, videoBody.reference_audio_urls)
+ videoBody.prompt = voices.prompt
+ videoBody.reference_audio_urls = voices.audioUrls
+ if (voices.hasVoiceReferences && videoBody.generate_audio === false) return badRequest(c, '引用角色音色时需要开启视频声音')
+ const referenceError = validateVideoRequest(videoBody, effectiveConfig?.provider)
+ if (referenceError) return badRequest(c, referenceError)
}
logTaskStart('TaskAPI', 'generate', {
diff --git a/backend/src/routes/upload.ts b/backend/src/routes/upload.ts
index aaec2e4..9a8053d 100644
--- a/backend/src/routes/upload.ts
+++ b/backend/src/routes/upload.ts
@@ -1,6 +1,8 @@
import { Hono } from 'hono'
+import { bodyLimit } from 'hono/body-limit'
import { success, badRequest } from '../utils/response.js'
import { saveUploadedFile, generateImageThumb } from '../utils/storage.js'
+import { AUDIO_MAX_BYTES, AudioUploadError, saveAudioUpload } from '../utils/audio-upload.js'
const app = new Hono()
@@ -24,10 +26,6 @@ const VIDEO_EXT = new Set(['.mp4', '.mov', '.webm', '.m4v'])
const VIDEO_MIME = new Set(['video/mp4', 'video/quicktime', 'video/webm', 'video/x-m4v'])
const VIDEO_MAX = 50 * 1024 * 1024 // 50MB
-const AUDIO_EXT = new Set(['.mp3', '.wav', '.m4a', '.aac'])
-const AUDIO_MIME = new Set(['audio/mpeg', 'audio/wav', 'audio/x-wav', 'audio/mp4', 'audio/x-m4a', 'audio/aac'])
-const AUDIO_MAX = 20 * 1024 * 1024 // 20MB
-
function extOf(name: string): string {
const i = name.lastIndexOf('.')
return i >= 0 ? name.slice(i).toLowerCase() : ''
@@ -64,6 +62,16 @@ async function saveMediaUpload(
app.post('/video', async (c) => saveMediaUpload(c, 'video', VIDEO_EXT, VIDEO_MIME, VIDEO_MAX))
// POST /upload/audio — 参考音频上传(Seedance 多模态参考用)
-app.post('/audio', async (c) => saveMediaUpload(c, 'audio', AUDIO_EXT, AUDIO_MIME, AUDIO_MAX))
+app.post('/audio', bodyLimit({ maxSize: AUDIO_MAX_BYTES + 64 * 1024,
+ onError: c => badRequest(c, '音频文件大小不能超过 20MB'),
+}), async (c) => {
+ try {
+ const body = await c.req.parseBody()
+ return success(c, await saveAudioUpload(body.file))
+ } catch (error) {
+ if (error instanceof AudioUploadError) return badRequest(c, error.message)
+ throw error
+ }
+})
export default app
diff --git a/backend/src/services/adapters/minimax-video.ts b/backend/src/services/adapters/minimax-video.ts
index 6090fbc..37b1bcf 100644
--- a/backend/src/services/adapters/minimax-video.ts
+++ b/backend/src/services/adapters/minimax-video.ts
@@ -81,7 +81,7 @@ export class MiniMaxVideoAdapter implements VideoProviderAdapter {
content.push({ type: 'video_url', video_url: { url }, role: 'reference_video' })
}
for (const url of refAudios) {
- content.push({ type: 'audio_url', audio_url: { url } })
+ content.push({ type: 'audio_url', audio_url: { url }, role: 'reference_audio' })
}
const body: any = {
diff --git a/backend/src/services/character-voice-references.ts b/backend/src/services/character-voice-references.ts
new file mode 100644
index 0000000..1b1ccfc
--- /dev/null
+++ b/backend/src/services/character-voice-references.ts
@@ -0,0 +1,53 @@
+import { and, eq, isNull } from 'drizzle-orm'
+import { db, schema } from '../db/index.js'
+
+type VoiceCharacter = { name: string; voiceAudioUrl: string | null; bound?: boolean }
+const voiceToken = /#[^\s#@,。!?;、:,.!?;:「」“”"'()()[\]{}<>]{1,100}/u
+
+/** Only explicitly mentioned voices are sent; indices follow the audio array, not image order. */
+export function resolveVoiceMentions(prompt: string, characters: VoiceCharacter[], existingAudio: string[] = []) {
+ const audioUrls = [...new Set(existingAudio.map(url => url.trim()))]
+ const names = [...new Set(characters.map(c => c.name).filter(Boolean))].sort((a, b) => b.length - a.length)
+ let hasVoiceReferences = false
+ let result = ''
+ for (let pos = 0; pos < prompt.length;) {
+ if (prompt[pos] !== '#') { result += prompt[pos++]; continue }
+ const tail = prompt.slice(pos)
+ const name = names.find(name => tail.startsWith(`#${name}`))
+ if (name === undefined) {
+ const unknown = tail.match(voiceToken)
+ // Numeric shot labels such as #01 are not voice mentions.
+ if (unknown?.index === 0 && !/^#\d+$/.test(unknown[0])) throw new Error(`音色引用「${unknown[0]}」无效,请选择当前分镜已绑定角色的音色`)
+ result += prompt[pos++]
+ continue
+ }
+ const matches = characters.filter(c => c.name === name && c.bound !== false)
+ if (!matches.length) throw new Error(`音色引用「#${name}」无效,该角色未绑定到当前分镜`)
+ if (matches.length > 1) throw new Error(`当前分镜存在同名角色「${name}」,请先修改角色名称再引用音色`)
+ const url = matches[0].voiceAudioUrl?.trim()
+ if (!url) throw new Error(`角色「${name}」尚未上传音色或音色已移除,请重新上传后生成`)
+ let index = audioUrls.indexOf(url)
+ if (index < 0) { index = audioUrls.length; audioUrls.push(url) }
+ hasVoiceReferences = true
+ result += `${name}(音色参考音频${index + 1})`
+ pos += name.length + 1
+ // Keep saved prompts using the previous #Name音色 spelling functional.
+ if (prompt.slice(pos).startsWith('音色')) pos += 2
+ }
+ return { prompt: result, audioUrls, hasVoiceReferences }
+}
+
+export async function resolveStoryboardVoiceReferences(prompt: string, storyboardId: number | undefined, existingAudio: string[]) {
+ if (!prompt.includes('#')) return { prompt, audioUrls: existingAudio, hasVoiceReferences: false }
+ if (!storyboardId) {
+ if ([...prompt.matchAll(new RegExp(voiceToken, 'gu'))].some(match => !/^#\d+$/.test(match[0]))) throw new Error('引用角色音色时必须指定当前分镜')
+ return { prompt, audioUrls: existingAudio, hasVoiceReferences: false }
+ }
+ const characters = await db.select({ name: schema.characters.name, voiceAudioUrl: schema.characters.voiceAudioUrl, boundId: schema.storyboardCharacters.characterId })
+ .from(schema.storyboards)
+ .innerJoin(schema.episodes, eq(schema.storyboards.episodeId, schema.episodes.id))
+ .innerJoin(schema.characters, eq(schema.characters.dramaId, schema.episodes.dramaId))
+ .leftJoin(schema.storyboardCharacters, and(eq(schema.storyboardCharacters.storyboardId, schema.storyboards.id), eq(schema.storyboardCharacters.characterId, schema.characters.id)))
+ .where(and(eq(schema.storyboards.id, storyboardId), isNull(schema.storyboards.deletedAt), isNull(schema.episodes.deletedAt), isNull(schema.characters.deletedAt)))
+ return resolveVoiceMentions(prompt, characters.map(c => ({ ...c, bound: c.boundId !== null })), existingAudio)
+}
diff --git a/backend/src/services/generation.ts b/backend/src/services/generation.ts
index a362ce1..4b11248 100644
--- a/backend/src/services/generation.ts
+++ b/backend/src/services/generation.ts
@@ -7,6 +7,7 @@ import { and, desc, eq, gt } from 'drizzle-orm'
import fs from 'node:fs'
import { getAbsolutePath } from '../utils/storage.js'
import { downloadVideo, safeTaskError } from './video-download.js'
+import { prepareReferenceAudio, resolveReferenceAudioUrls } from './reference-audio.js'
import { getActiveConfig, getConfigById } from './ai.js'
import { now } from '../utils/response.js'
import { downloadFile, fetchImageAsCompressedDataUrl, generateImageThumb, readImageAsCompressedDataUrl, readImageAsDataUrl, saveBase64Image } from '../utils/storage.js'
@@ -127,6 +128,8 @@ export async function generateVideo(params: GenerateVideoParams): Promise<number
: await getActiveConfig('video')
if (!config) throw new Error('未配置视频模型,请先到「设置」页添加并启用 AI 服务')
+ params.referenceAudioUrls = await prepareReferenceAudio(params.referenceAudioUrls, config.provider)
+
const id = await createTask('video', config, {
storyboardId: params.storyboardId,
dramaId: params.dramaId,
@@ -247,9 +250,9 @@ async function processTask(id: number, config: AIConfig) {
const resolvedFirstFrameUrl = await normalizeVideoReferenceUrl(params.firstFrameUrl)
const resolvedLastFrameUrl = await normalizeVideoReferenceUrl(params.lastFrameUrl)
const resolvedReferenceImageUrls = await normalizeVideoReferenceUrls(params.referenceImageUrls)
- // 参考视频/音频文件较大,不适合 dataURL 内联,需解析为公网可访问 URL
+ // 视频使用公网地址;短音色按厂商能力使用 data URL 或公网地址。
const resolvedReferenceVideoUrls = resolvePublicMediaUrls(params.referenceVideoUrls, 'video')
- const resolvedReferenceAudioUrls = resolvePublicMediaUrls(params.referenceAudioUrls, 'audio')
+ const resolvedReferenceAudioUrls = await resolveReferenceAudioUrls(params.referenceAudioUrls, config.provider)
const resolvedReferenceFileUrl = resolvePublicMediaUrl(params.referenceFileUrl, 'file')
;({ url, method, headers, body } = adapter.buildGenerateRequest(config, {
id: record.id,
diff --git a/backend/src/services/reference-audio.ts b/backend/src/services/reference-audio.ts
new file mode 100644
index 0000000..1267b36
--- /dev/null
+++ b/backend/src/services/reference-audio.ts
@@ -0,0 +1,95 @@
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import { createHash, randomUUID } from 'node:crypto'
+import { STORAGE_ROOT } from '../utils/paths.js'
+import { ffmpeg, checkFfmpegSuite } from '../utils/ffmpeg.js'
+
+const MAX_BYTES = 15 * 1024 * 1024
+const INLINE_PROVIDERS = new Set(['volcengine', 'minimax'])
+
+function localAudioPath(url: string): string | null {
+ if (!/^\/?static\//.test(url)) return null
+ const file = path.resolve(STORAGE_ROOT, url.replace(/^\/?static\//, ''))
+ if (!file.startsWith(path.resolve(STORAGE_ROOT) + path.sep)) throw new Error('无效的本地音色路径')
+ return file
+}
+
+function publicAudioUrl(url: string): string {
+ const base = (process.env.PUBLIC_BASE_URL || '').trim().replace(/\/+$/, '')
+ if (!/^https?:\/\//.test(base)) {
+ throw new Error('当前视频模型需要公网音色地址。请配置后端 PUBLIC_BASE_URL,或切换至支持本地音色的 Seedance 2.0 / MiniMax H3。')
+ }
+ return `${base}/${url.replace(/^\//, '')}`
+}
+
+/** Validate before creating a paid task. M4A/AAC samples are converted without trimming the voice. */
+export async function prepareReferenceAudio(refs: string[] | undefined, provider: string): Promise<string[]> {
+ const urls = [...new Set((refs || []).map(url => url.trim()).filter(Boolean))]
+ const minSeconds = provider === 'aliyun' ? 1 : 2
+ let durationSum = 0
+ const prepared: string[] = []
+ for (const url of urls) {
+ const file = localAudioPath(url)
+ if (!file) { prepared.push(url); continue }
+ if (!INLINE_PROVIDERS.has(provider)) publicAudioUrl(url)
+ let size: number
+ try { size = (await fs.stat(file)).size } catch { throw new Error('角色音色文件不存在,请重新上传音色') }
+ const ext = path.extname(file).toLowerCase()
+ if (!['.mp3', '.wav', '.m4a', '.aac'].includes(ext)) throw new Error('参考音色仅支持 MP3、WAV、M4A 或 AAC')
+ if (size > 20 * 1024 * 1024) throw new Error('音色文件过大,请上传不超过 20MB 的音频')
+ const suite = await checkFfmpegSuite()
+ if (!suite.ffprobe) throw new Error('音色校验需要 FFprobe,请修复后端 FFprobe 配置后重试')
+ const duration = await new Promise<number>((resolve, reject) => {
+ ffmpeg.ffprobe(file, (error, metadata) => {
+ if (error || !metadata.streams.some(stream => stream.codec_type === 'audio')) {
+ reject(new Error('无法读取音色文件,请上传有效的人声音频')); return
+ }
+ resolve(Number(metadata.format.duration))
+ })
+ })
+ if (!Number.isFinite(duration) || duration < minSeconds || duration > 15) {
+ throw new Error(`当前模型要求每段参考音色为 ${minSeconds}–15 秒,请裁剪或重新上传`)
+ }
+ durationSum += duration
+ if (durationSum > 15.05) throw new Error('本次引用的音色总时长不能超过 15 秒,请减少引用或缩短音色样本')
+ if (ext === '.wav' || ext === '.mp3') {
+ if (size > MAX_BYTES) throw new Error('当前模型要求每段参考音色不超过 15MB,请压缩或重新上传')
+ prepared.push(url.replace(/^\//, ''))
+ continue
+ }
+ if (!suite.ffmpeg) throw new Error('M4A / AAC 音色需要 FFmpeg 转换,请修复配置或上传 WAV / MP3')
+ const hash = createHash('sha256').update(await fs.readFile(file)).digest('hex')
+ const relative = `static/voice-references/${hash}.wav`
+ const target = localAudioPath(relative)!
+ await fs.mkdir(path.dirname(target), { recursive: true })
+ try { await fs.access(target) } catch {
+ const temporary = `${target}.${randomUUID()}.wav`
+ try {
+ await new Promise<void>((resolve, reject) => {
+ ffmpeg(file).noVideo().audioChannels(1).audioFrequency(48000).audioCodec('pcm_s16le')
+ .format('wav').on('end', () => resolve()).on('error', () => reject(new Error('音色转换失败,请改用 WAV / MP3 文件'))).save(temporary)
+ })
+ await fs.rename(temporary, target)
+ } finally { await fs.rm(temporary, { force: true }) }
+ }
+ prepared.push(relative)
+ }
+ return prepared
+}
+
+/** Seedance 2.0 / MiniMax H3 accept audio data URLs; Wan requires a public/OSS URL.
+ * https://docs.volcengine.com/docs/ark/create-video-generation-task-api
+ * https://platform.minimax.cn/docs/api-reference/video-generation-v2-create
+ * https://help.aliyun.com/en/model-studio/wan3-video-generation-api-reference
+ */
+export async function resolveReferenceAudioUrls(refs: string[] | undefined, provider: string): Promise<string[]> {
+ return Promise.all((refs || []).map(async url => {
+ const file = localAudioPath(url)
+ if (!file) return url
+ if (!INLINE_PROVIDERS.has(provider)) return publicAudioUrl(url)
+ const bytes = await fs.readFile(file)
+ if (bytes.length > MAX_BYTES) throw new Error('参考音色不能超过 15MB')
+ const format = path.extname(file).toLowerCase() === '.mp3' ? 'mp3' : 'wav'
+ return `data:audio/${format};base64,${bytes.toString('base64')}`
+ }))
+}
diff --git a/backend/src/utils/audio-upload.ts b/backend/src/utils/audio-upload.ts
new file mode 100644
index 0000000..e36df84
--- /dev/null
+++ b/backend/src/utils/audio-upload.ts
@@ -0,0 +1,22 @@
+import { saveUploadedFile } from './storage.js'
+
+export const AUDIO_MAX_BYTES = 20 * 1024 * 1024
+const AUDIO_EXT = new Set(['.mp3', '.wav', '.m4a', '.aac'])
+const AUDIO_MIME = new Set(['audio/mpeg', 'audio/mp3', 'audio/wav', 'audio/x-wav', 'audio/wave', 'audio/vnd.wave', 'audio/mp4', 'audio/x-m4a', 'audio/aac', 'audio/x-aac'])
+
+export class AudioUploadError extends Error {}
+
+/** 音色与视频参考音频共用上传规则,文件名仅用于显示,磁盘使用 UUID。 */
+export async function saveAudioUpload(file: unknown) {
+ if (!(file instanceof File)) throw new AudioUploadError('文件必填')
+ const name = file.name.split(/[\\/]/).pop()!.replace(/[\u0000-\u001f\u007f]/g, '').slice(0, 255)
+ const ext = file.name.slice(file.name.lastIndexOf('.')).toLowerCase()
+ const mime = file.type.toLowerCase()
+ if (!AUDIO_EXT.has(ext) || (mime && mime !== 'application/octet-stream' && !AUDIO_MIME.has(mime))) {
+ throw new AudioUploadError('仅支持 .mp3/.wav/.m4a/.aac 格式的音频文件')
+ }
+ if (!file.size) throw new AudioUploadError('音频文件不能为空')
+ if (file.size > AUDIO_MAX_BYTES) throw new AudioUploadError('音频文件大小不能超过 20MB')
+ const path = await saveUploadedFile(await file.arrayBuffer(), 'uploads', `voice${ext}`)
+ return { path, name, url: `/${path}` }
+}
diff --git a/backend/src/utils/task-logger.ts b/backend/src/utils/task-logger.ts
index 08068fa..fa22bd6 100644
--- a/backend/src/utils/task-logger.ts
+++ b/backend/src/utils/task-logger.ts
@@ -42,6 +42,7 @@ function formatMeta(meta?: Record<string, unknown>) {
}
export function redactUrl(rawUrl: string) {
+ if (rawUrl.startsWith('data:')) return `${rawUrl.slice(0, rawUrl.indexOf(',') + 1)}<omitted>`
try {
const url = new URL(rawUrl)
for (const key of [...url.searchParams.keys()]) {
diff --git a/backend/tests/character-voice.test.ts b/backend/tests/character-voice.test.ts
new file mode 100644
index 0000000..c49a873
--- /dev/null
+++ b/backend/tests/character-voice.test.ts
@@ -0,0 +1,151 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import Database from 'better-sqlite3'
+import { Hono } from 'hono'
+import { serveStatic } from '@hono/node-server/serve-static'
+import { eq } from 'drizzle-orm'
+import { initSqliteSchema } from '../src/db/sqlite-schema.ts'
+import { requestLogger } from '../src/middleware/logger.ts'
+
+const root = await fs.mkdtemp(path.join(os.tmpdir(), 'huobao-character-voice-'))
+process.env.SQLITE_PATH = path.join(root, 'test.sqlite3')
+process.env.HUOBAO_DATA_DIR = root
+process.env.STORAGE_PATH = path.join(root, 'static')
+process.env.MYSQL_AUTO_IMPORT = 'false'
+const { db, schema } = await import('../src/db/index.ts')
+const app = new Hono()
+app.route('/characters', (await import('../src/routes/characters.ts')).default)
+app.route('/episodes', (await import('../src/routes/episodes.ts')).default)
+app.route('/dramas', (await import('../src/routes/dramas.ts')).default)
+app.route('/upload', (await import('../src/routes/upload.ts')).default)
+app.use('/static/*', serveStatic({ root }))
+const ts = new Date().toISOString()
+const drama = db.insert(schema.dramas).values({ title: 'Voice test', createdAt: ts, updatedAt: ts }).returning().get()
+const character = () => db.insert(schema.characters).values({ dramaId: drama.id, name: '角色', appearance: '原外貌', imageUrl: 'static/images/portrait.png', createdAt: ts, updatedAt: ts }).returning().get()
+const row = (id: number) => db.select().from(schema.characters).where(eq(schema.characters.id, id)).get()!
+const sample = Buffer.alloc(48)
+sample.write('RIFF'); sample.writeUInt32LE(40, 4); sample.write('WAVEfmt ', 8)
+sample.writeUInt32LE(16, 16); sample.writeUInt16LE(1, 20); sample.writeUInt16LE(1, 22)
+sample.writeUInt32LE(8000, 24); sample.writeUInt32LE(16000, 28)
+sample.writeUInt16LE(2, 32); sample.writeUInt16LE(16, 34); sample.write('data', 36); sample.writeUInt32LE(4, 40)
+async function upload(id: number, name = '角色音色.wav', data: Uint8Array = sample, type = 'audio/wav') {
+ const body = new FormData()
+ body.set('file', new File([data], name, { type }))
+ return app.request(`/characters/${id}/voice`, { method: 'POST', body })
+}
+
+test('existing SQLite characters gain nullable voice fields without losing assets; migration is idempotent', () => {
+ const old = new Database(':memory:')
+ try {
+ old.exec(`CREATE TABLE characters (id INTEGER PRIMARY KEY AUTOINCREMENT, drama_id INTEGER NOT NULL,
+ name TEXT NOT NULL, role TEXT, description TEXT, appearance TEXT, styling TEXT, final_prompt TEXT,
+ personality TEXT, image_url TEXT, reference_images TEXT, seed_value TEXT, sort_order INTEGER,
+ local_path TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, deleted_at TEXT);
+ INSERT INTO characters (drama_id, name, image_url, created_at, updated_at) VALUES (1, '已有角色', 'portrait.png', 'old', 'old')`)
+ initSqliteSchema(old)
+ initSqliteSchema(old)
+ assert.deepEqual(old.prepare('SELECT name, image_url, voice_audio_url, voice_audio_name FROM characters').get(),
+ { name: '已有角色', image_url: 'portrait.png', voice_audio_url: null, voice_audio_name: null })
+ } finally { old.close() }
+})
+
+test('uploaded voice persists, can be served, and is shared by the project and multiple episodes', async () => {
+ const char = character()
+ const result = await upload(char.id)
+ assert.equal(result.status, 200)
+ const { data } = await result.json()
+ assert.equal(data.voice_audio_name, '角色音色.wav')
+ assert.match(data.voice_audio_url, /^static\/uploads\/[\w-]+\.wav$/)
+ const disk = new Database(process.env.SQLITE_PATH!, { readonly: true })
+ try { assert.equal((disk.prepare('SELECT voice_audio_url FROM characters WHERE id = ?').get(char.id) as any).voice_audio_url, data.voice_audio_url) }
+ finally { disk.close() }
+ const audio = await app.request(`/${data.voice_audio_url}`)
+ assert.equal(audio.status, 200)
+ assert.deepEqual(Buffer.from(await audio.arrayBuffer()), sample)
+ for (const number of [1, 2]) {
+ const episode = db.insert(schema.episodes).values({ dramaId: drama.id, episodeNumber: number, title: '测试集', createdAt: ts, updatedAt: ts }).returning().get()
+ db.insert(schema.episodeCharacters).values({ episodeId: episode.id, characterId: char.id, createdAt: ts }).run()
+ const response = await app.request(`/episodes/${episode.id}/characters`)
+ assert.equal((await response.json()).data[0].voice_audio_url, data.voice_audio_url)
+ }
+ const project = await app.request(`/dramas/${drama.id}`)
+ assert.equal((await project.json()).data.characters.find((c: any) => c.id === char.id).voice_audio_name, '角色音色.wav')
+ await app.request(`/characters/${char.id}`, { method: 'PUT', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ appearance: '新外貌' }) })
+ assert.equal(row(char.id).voiceAudioUrl, data.voice_audio_url)
+})
+
+test('replacement and removal change only the voice binding', async () => {
+ const char = character()
+ await upload(char.id)
+ const original = row(char.id).voiceAudioUrl
+ const replacement = await upload(char.id, '第二段.WAV', sample, 'audio/x-wav')
+ assert.equal(replacement.status, 200)
+ assert.notEqual(row(char.id).voiceAudioUrl, original)
+ assert.equal(row(char.id).voiceAudioName, '第二段.WAV')
+ const removed = await app.request(`/characters/${char.id}/voice`, { method: 'DELETE' })
+ assert.equal(removed.status, 200)
+ assert.deepEqual((await removed.json()).data, { voice_audio_url: null, voice_audio_name: null })
+ assert.equal(row(char.id).voiceAudioUrl, null)
+ assert.equal(row(char.id).appearance, '原外貌')
+ assert.equal(row(char.id).imageUrl, char.imageUrl)
+})
+
+test('invalid, missing, empty and oversized uploads retain the current voice', async () => {
+ const char = character()
+ await upload(char.id)
+ const original = row(char.id).voiceAudioUrl
+ const cases: [string, Uint8Array, string][] = [
+ ['image.png', sample, 'image/png'], ['fake.wav', sample, 'text/html'],
+ ['empty.wav', new Uint8Array(), 'audio/wav'],
+ ['large.wav', new Uint8Array(20 * 1024 * 1024 + 1), 'audio/wav'],
+ ['too-large.wav', new Uint8Array(21 * 1024 * 1024), 'audio/wav'],
+ ]
+ for (const [name, bytes, type] of cases) {
+ assert.equal((await upload(char.id, name, bytes, type)).status, 400, name)
+ assert.equal(row(char.id).voiceAudioUrl, original)
+ }
+ assert.equal((await app.request(`/characters/${char.id}/voice`, { method: 'POST', body: new FormData() })).status, 400)
+})
+
+test('missing and deleted characters cannot acquire audio files', async () => {
+ const before = await fs.readdir(path.join(root, 'static/uploads'))
+ assert.equal((await upload(999999)).status, 404)
+ const char = character()
+ db.update(schema.characters).set({ deletedAt: ts }).where(eq(schema.characters.id, char.id)).run()
+ assert.equal((await upload(char.id)).status, 404)
+ assert.equal((await app.request(`/characters/${char.id}/voice`, { method: 'DELETE' })).status, 404)
+ assert.deepEqual(await fs.readdir(path.join(root, 'static/uploads')), before)
+})
+
+test('generic reference audio uploads still use the shared file contract', async () => {
+ const body = new FormData()
+ body.set('file', new File([sample], 'reference.wav', { type: 'audio/wav' }))
+ const result = await app.request('/upload/audio', { method: 'POST', body })
+ assert.equal(result.status, 200)
+ const { data } = await result.json()
+ assert.equal(data.url, `/${data.path}`)
+ assert.equal((await app.request(data.url)).status, 200)
+})
+
+test('request logging preserves the upload stream and omits audio contents', async t => {
+ const messages: string[] = []
+ const logger = t.mock.method(console, 'log', (...args: unknown[]) => messages.push(args.join(' ')))
+ try {
+ const logged = new Hono()
+ logged.use('*', requestLogger)
+ logged.route('/', app)
+ const char = character()
+ const body = new FormData()
+ body.set('file', new File([sample], 'private-recording.wav', { type: 'audio/wav' }))
+ const result = await logged.request(`/characters/${char.id}/voice`, { method: 'POST', body })
+ assert.equal(result.status, 200)
+ assert.equal(row(char.id).voiceAudioName, 'private-recording.wav')
+ assert.match(messages.join('\n'), /POST/)
+ assert.doesNotMatch(messages.join('\n'), /private-recording|RIFF|body:|Content-Disposition/)
+ } finally { logger.mock.restore() }
+})
+
+test.after(async () => { db.$client.close(); await fs.rm(root, { recursive: true, force: true }) })
diff --git a/backend/tests/remove-audio-tts-structure.test.mjs b/backend/tests/remove-audio-tts-structure.test.mjs
index c488fed..2f64c2c 100644
--- a/backend/tests/remove-audio-tts-structure.test.mjs
+++ b/backend/tests/remove-audio-tts-structure.test.mjs
@@ -36,12 +36,12 @@ test('backend removes audio service providers, TTS adapters, and voice routes',
assert.doesNotMatch(read('src/services/adapters/volcengine-video.ts'), /generate_audio:\s*false/)
})
-test('backend removes TTS endpoints and audio-specific schema fields', () => {
+test('backend keeps legacy TTS fields and endpoints removed while allowing uploaded voice samples', () => {
const episodes = read('src/routes/episodes.ts')
const storyboards = read('src/routes/storyboards.ts')
const characters = read('src/routes/characters.ts')
const schema = read('src/db/schema.ts')
- const mysqlSchema = read('src/db/mysql-schema.ts')
+ const sqliteSchema = read('src/db/sqlite-schema.ts')
assert.doesNotMatch(episodes, /audio_config_id/)
assert.doesNotMatch(episodes, /assign_voices/)
@@ -56,12 +56,10 @@ test('backend removes TTS endpoints and audio-specific schema fields', () => {
assert.doesNotMatch(schema, /voiceProvider/)
assert.doesNotMatch(schema, /ttsAudioUrl/)
assert.doesNotMatch(schema, /aiVoices/)
- assert.doesNotMatch(mysqlSchema, /audio_config_id/)
- assert.doesNotMatch(mysqlSchema, /voice_style/)
- assert.doesNotMatch(mysqlSchema, /voice_sample_url/)
- assert.doesNotMatch(mysqlSchema, /voice_provider/)
- assert.doesNotMatch(mysqlSchema, /tts_audio_url/)
- // 无 ai_voices 表定义;仅保留启动时的孤儿表清理 DROP
- assert.doesNotMatch(mysqlSchema, /CREATE TABLE IF NOT EXISTS ai_voices/)
- assert.match(mysqlSchema, /DROP TABLE IF EXISTS `ai_voices`/)
+ assert.doesNotMatch(sqliteSchema, /audio_config_id/)
+ assert.doesNotMatch(sqliteSchema, /voice_style/)
+ assert.doesNotMatch(sqliteSchema, /voice_sample_url/)
+ assert.doesNotMatch(sqliteSchema, /voice_provider/)
+ assert.doesNotMatch(sqliteSchema, /tts_audio_url/)
+ assert.doesNotMatch(sqliteSchema, /CREATE TABLE IF NOT EXISTS ai_voices/)
})
diff --git a/backend/tests/seedance2-video-modes.test.mjs b/backend/tests/seedance2-video-modes.test.mjs
index 91edbc9..9945f70 100644
--- a/backend/tests/seedance2-video-modes.test.mjs
+++ b/backend/tests/seedance2-video-modes.test.mjs
@@ -70,13 +70,15 @@ test('video resolution is fixed per episode, editable, and locked into video tas
test('upload route exposes validated video and audio endpoints', () => {
const route = read('src/routes/upload.ts')
+ const audioUpload = read('src/utils/audio-upload.ts')
assert.match(route, /app\.post\('\/video'/)
assert.match(route, /app\.post\('\/audio'/)
assert.match(route, /VIDEO_EXT = new Set\(\['\.mp4', '\.mov', '\.webm', '\.m4v'\]\)/)
- assert.match(route, /AUDIO_EXT = new Set\(\['\.mp3', '\.wav', '\.m4a', '\.aac'\]\)/)
+ assert.match(route, /saveAudioUpload/)
+ assert.match(audioUpload, /AUDIO_EXT = new Set\(\['\.mp3', '\.wav', '\.m4a', '\.aac'\]\)/)
assert.match(route, /50 \* 1024 \* 1024/)
- assert.match(route, /20 \* 1024 \* 1024/)
+ assert.match(audioUpload, /20 \* 1024 \* 1024/)
})
test('tasks route validates reference-mode requirements for video tasks', () => {
diff --git a/backend/tests/video-prompt-voices.test.ts b/backend/tests/video-prompt-voices.test.ts
new file mode 100644
index 0000000..321ae42
--- /dev/null
+++ b/backend/tests/video-prompt-voices.test.ts
@@ -0,0 +1,130 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs/promises'
+import path from 'node:path'
+import os from 'node:os'
+import { eq } from 'drizzle-orm'
+
+const root = await fs.mkdtemp(path.join(os.tmpdir(), 'huobao-prompt-voices-'))
+process.env.SQLITE_PATH = path.join(root, 'test.sqlite3')
+process.env.HUOBAO_DATA_DIR = root
+process.env.STORAGE_PATH = path.join(root, 'static')
+process.env.WORKSPACE_PATH = path.join(root, 'workspace')
+process.env.MYSQL_AUTO_IMPORT = 'false'
+await fs.cp(new URL('../workspace/', import.meta.url), process.env.WORKSPACE_PATH, { recursive: true })
+const { db, schema } = await import('../src/db/index.ts')
+const { storyboardTools } = await import('../src/agents/tools/storyboard-tools.ts')
+const { buildAgentRequestContext } = await import('../src/agents/context.ts')
+const { agentRegistry } = await import('../src/agents/index.ts')
+const { resolveStoryboardVoiceReferences } = await import('../src/services/character-voice-references.ts')
+const ts = '2026-01-01T00:00:00.000Z'
+const timestamps = { createdAt: ts, updatedAt: ts }
+const drama = db.insert(schema.dramas).values({ title: 'Prompt voices', ...timestamps }).returning().get()
+const ep = db.insert(schema.episodes).values({ dramaId: drama.id, episodeNumber: 1, title: 'Test', scriptContent: '林小雨说:「早上好。」陈默点头。', ...timestamps }).returning().get()
+const createCharacter = (name: string, voice = true) => {
+ const c = db.insert(schema.characters).values({ name, dramaId: drama.id, voiceAudioUrl: voice ? `static/uploads/private-${name}.wav` : null, ...timestamps }).returning().get()
+ db.insert(schema.episodeCharacters).values({ episodeId: ep.id, characterId: c.id, createdAt: ts }).run()
+ return c
+}
+const speaker = createCharacter('林小雨')
+const silent = createCharacter('陈默')
+const noVoice = createCharacter('小白', false)
+const unbound = createCharacter('未出场')
+const deleted = createCharacter('已删除')
+db.update(schema.characters).set({ deletedAt: ts }).where(eq(schema.characters.id, deleted.id)).run()
+const sb = db.insert(schema.storyboards).values({ episodeId: ep.id, storyboardNumber: 1,
+ description: '【镜头1】林小雨说:「早上好。」陈默点头。', atmosphere: '晨光', duration: 6, videoPrompt: '原提示词', ...timestamps }).returning().get()
+for (const c of [speaker, silent, noVoice, deleted]) db.insert(schema.storyboardCharacters).values({ storyboardId: sb.id, characterId: c.id }).run()
+const requestContext = buildAgentRequestContext({ dramaId: drama.id, episodeId: ep.id })
+const call = (name: keyof typeof storyboardTools, input: any) => (storyboardTools[name].execute as any)(input, { requestContext })
+const row = () => db.select().from(schema.storyboards).where(eq(schema.storyboards.id, sb.id)).get()!
+
+test('prompt context exposes exact voice tokens and scopes each shot to bound, live characters', async () => {
+ const context = await call('readStoryboardContext', {})
+ assert.equal(context.characters.find((c: any) => c.id === speaker.id).voice_reference, '#林小雨')
+ assert.equal(context.characters.find((c: any) => c.id === noVoice.id).has_voice, false)
+ assert.equal(context.characters.find((c: any) => c.id === noVoice.id).voice_reference, null)
+ assert.equal(context.characters.some((c: any) => c.id === deleted.id), false)
+ assert.equal(context.characters.some((c: any) => c.id === unbound.id), true)
+ const voices = context.existing_storyboards[0].available_voice_references
+ assert.deepEqual(voices, [
+ { character_id: speaker.id, name: speaker.name, voice_reference: '#林小雨' },
+ { character_id: silent.id, name: silent.name, voice_reference: '#陈默' },
+ ])
+ assert.doesNotMatch(JSON.stringify(context), /static\/uploads|private-/)
+})
+
+test('single, batch and initial-breakdown agents get voice rules in every content language', async () => {
+ for (const language of ['zh', 'en', 'ja', 'ko'] as const) {
+ const rc = buildAgentRequestContext({ dramaId: drama.id, episodeId: ep.id, language })
+ for (const type of ['prompt_generator', 'storyboard_breaker']) {
+ const instructions = String(await agentRegistry[type].getInstructions({ requestContext: rc }))
+ assert.match(instructions, /available_voice_references/)
+ assert.match(instructions, /voice_reference/)
+ assert.match(instructions, /#(?:林小雨|Alex|太郎|민수)/)
+ if (language !== 'zh') assert.match(instructions, /copy the complete voice_reference value exactly/)
+ }
+ }
+ const extractor = String(await agentRegistry.extractor.getInstructions({ requestContext }))
+ assert.doesNotMatch(extractor, /available_voice_references/)
+})
+
+test('custom and missing prompt templates retain the runtime voice rules', async () => {
+ const file = path.join(process.env.WORKSPACE_PATH!, 'prompts/prompt_generator.md')
+ const original = await fs.readFile(file, 'utf8')
+ try {
+ await fs.writeFile(file, '---\nname: Custom\nmodel: ""\n---\nCustom prompt sentinel.')
+ const custom = String(await agentRegistry.prompt_generator.getInstructions({ requestContext }))
+ assert.match(custom, /Custom prompt sentinel/)
+ assert.match(custom, /available_voice_references/)
+ await fs.rm(file)
+ const fallback = String(await agentRegistry.prompt_generator.getInstructions({ requestContext }))
+ assert.match(fallback, /available_voice_references/)
+ } finally { await fs.writeFile(file, original) }
+})
+
+test('a generated voice binding saves editable tokens and resolves to the corresponding video audio', async () => {
+ const prompt = '0-3秒:@林小雨抬头,#林小雨轻声说:「早上好。」\n3-6秒:@陈默点头。'
+ await call('updateStoryboard', { storyboard_id: sb.id, video_prompt: prompt })
+ assert.equal(row().videoPrompt, prompt)
+ assert.equal(row().description, sb.description)
+ assert.equal(row().atmosphere, sb.atmosphere)
+ assert.equal(row().duration, sb.duration)
+ const resolved = await resolveStoryboardVoiceReferences(row().videoPrompt!, sb.id, [])
+ assert.deepEqual(resolved.audioUrls, [speaker.voiceAudioUrl])
+ assert.equal(resolved.prompt, '0-3秒:@林小雨抬头,林小雨(音色参考音频1)轻声说:「早上好。」\n3-6秒:@陈默点头。')
+ assert.doesNotMatch(resolved.prompt, /#林小雨/)
+})
+
+test('invented, unbound, removed and missing voice references cannot replace a saved prompt', async () => {
+ const before = row().videoPrompt
+ for (const name of ['虚构角色', unbound.name, noVoice.name, deleted.name]) {
+ await assert.rejects(call('updateStoryboard', { storyboard_id: sb.id, video_prompt: `音色:#${name}` }), /音色/)
+ assert.equal(row().videoPrompt, before)
+ }
+ db.update(schema.characters).set({ voiceAudioUrl: null }).where(eq(schema.characters.id, speaker.id)).run()
+ try {
+ const context = await call('readStoryboardContext', {})
+ assert.equal(context.existing_storyboards[0].available_voice_references.some((v: any) => v.character_id === speaker.id), false)
+ await assert.rejects(call('updateStoryboard', { storyboard_id: sb.id, video_prompt: '#林小雨' }), /尚未上传/)
+ assert.equal(row().videoPrompt, before)
+ } finally { db.update(schema.characters).set({ voiceAudioUrl: speaker.voiceAudioUrl }).where(eq(schema.characters.id, speaker.id)).run() }
+})
+
+test('initial breakdown validates voices before replacement and persists valid bindings', async () => {
+ const before = row().videoPrompt
+ await assert.rejects(call('saveStoryboards', { replace_existing: true, storyboards: [
+ { shot_number: 1, character_ids: [speaker.id], video_prompt: '#虚构角色' },
+ ] }), /音色/)
+ assert.equal(row().videoPrompt, before)
+ await call('saveStoryboards', { storyboards: [
+ { shot_number: 2, character_ids: [speaker.id], description: '林小雨说:「再见。」', video_prompt: '0-3秒:@林小雨挥手,#林小雨说:「再见。」' },
+ { shot_number: 3, character_ids: [noVoice.id], description: '小白走路,无对白。', video_prompt: '@小白走路。' },
+ ] })
+ const shots = db.select().from(schema.storyboards).where(eq(schema.storyboards.episodeId, ep.id)).all()
+ const second = shots.find(s => s.storyboardNumber === 2)!
+ assert.deepEqual((await resolveStoryboardVoiceReferences(second.videoPrompt!, second.id, [])).audioUrls, [speaker.voiceAudioUrl])
+ assert.equal(shots.find(s => s.storyboardNumber === 3)!.videoPrompt, '@小白走路。')
+})
+
+test.after(async () => { db.$client.close(); await fs.rm(root, { recursive: true, force: true }) })
diff --git a/backend/tests/voice-reference-generation.test.ts b/backend/tests/voice-reference-generation.test.ts
new file mode 100644
index 0000000..da51874
--- /dev/null
+++ b/backend/tests/voice-reference-generation.test.ts
@@ -0,0 +1,202 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import fs from 'node:fs/promises'
+import os from 'node:os'
+import path from 'node:path'
+import http from 'node:http'
+import { eq } from 'drizzle-orm'
+
+const root = await fs.mkdtemp(path.join(os.tmpdir(), 'huobao-voice-reference-'))
+process.env.SQLITE_PATH = path.join(root, 'test.sqlite3')
+process.env.HUOBAO_DATA_DIR = root
+process.env.STORAGE_PATH = path.join(root, 'static')
+process.env.MYSQL_AUTO_IMPORT = 'false'
+delete process.env.PUBLIC_BASE_URL
+const { db, schema } = await import('../src/db/index.ts')
+const { resolveVoiceMentions } = await import('../src/services/character-voice-references.ts')
+const { prepareReferenceAudio, resolveReferenceAudioUrls } = await import('../src/services/reference-audio.ts')
+const { ffmpeg } = await import('../src/utils/ffmpeg.ts')
+const { logTaskPayload } = await import('../src/utils/task-logger.ts')
+const route = (await import('../src/routes/tasks.ts')).default
+const received: any[] = []
+const server = http.createServer(async (req, res) => {
+ const chunks = []
+ for await (const chunk of req) chunks.push(chunk)
+ received.push(JSON.parse(Buffer.concat(chunks).toString()))
+ // Stop after capturing the submission. No paid generation, polling or remote download.
+ res.writeHead(400, { 'Content-Type': 'application/json' }).end('{"error":{"message":"test capture complete"}}')
+})
+await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))
+const baseUrl = `http://127.0.0.1:${(server.address() as any).port}`
+const ts = new Date().toISOString()
+const timestamps = { createdAt: ts, updatedAt: ts }
+const drama = db.insert(schema.dramas).values({ title: '声音引用测试', ...timestamps }).returning().get()
+const episode = db.insert(schema.episodes).values({ dramaId: drama.id, episodeNumber: 1, title: '测试', ...timestamps }).returning().get()
+const configs = Object.fromEntries(['volcengine', 'minimax', 'aliyun'].map(provider => [provider,
+ db.insert(schema.aiServiceConfigs).values({ serviceType: 'video', provider, name: 'test', baseUrl, apiKey: 'mock-key',
+ model: JSON.stringify([provider === 'volcengine' ? 'doubao-seedance-2-0-mini-260615' : provider === 'minimax' ? 'MiniMax-H3' : 'wan3.0-video']), ...timestamps }).returning().get().id,
+]))
+function wav(seconds: number) {
+ const buffer = Buffer.alloc(44 + 16000 * seconds)
+ buffer.write('RIFF'); buffer.writeUInt32LE(buffer.length - 8, 4); buffer.write('WAVEfmt ', 8)
+ buffer.writeUInt32LE(16, 16); buffer.writeUInt16LE(1, 20); buffer.writeUInt16LE(1, 22)
+ buffer.writeUInt32LE(8000, 24); buffer.writeUInt32LE(16000, 28); buffer.writeUInt16LE(2, 32); buffer.writeUInt16LE(16, 34)
+ buffer.write('data', 36); buffer.writeUInt32LE(buffer.length - 44, 40)
+ return buffer
+}
+await fs.mkdir(path.join(root, 'static/uploads'), { recursive: true })
+for (const seconds of [1, 3, 9, 16]) await fs.writeFile(path.join(root, `static/uploads/${seconds}.wav`), wav(seconds))
+const char = (name: string, url: string | null = 'static/uploads/3.wav', dramaId = drama.id) =>
+ db.insert(schema.characters).values({ name, dramaId, voiceAudioUrl: url, ...timestamps }).returning().get()
+const shot = (...characters: { id: number }[]) => {
+ const sb = db.insert(schema.storyboards).values({ episodeId: episode.id, storyboardNumber: 1, ...timestamps }).returning().get()
+ for (const c of characters) db.insert(schema.storyboardCharacters).values({ storyboardId: sb.id, characterId: c.id }).run()
+ return sb.id
+}
+async function submit(storyboardId: number, prompt: string, extra = {}) {
+ return route.request('/', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({
+ type: 'video', storyboard_id: storyboardId, prompt, config_id: configs.volcengine,
+ reference_image_urls: ['https://example.test/portrait.png'], ...extra,
+ }) })
+}
+async function waitForSubmission(id: number) {
+ const end = Date.now() + 5000
+ while (Date.now() < end) {
+ const row = db.select().from(schema.sysTask).where(eq(schema.sysTask.id, id)).get()!
+ if (row.status === 'failed') return row
+ await new Promise(resolve => setTimeout(resolve, 10))
+ }
+ assert.fail('mock submission did not complete')
+}
+
+test('voice indices follow explicit audio then first appearance; repeats and shared samples are deduplicated', () => {
+ const resolved = resolveVoiceMentions('@小雨 使用#小雨姐姐,#小雨,#小雨姐姐', [
+ { name: '小雨', voiceAudioUrl: 'a.wav' }, { name: '小雨姐姐', voiceAudioUrl: 'b.wav' },
+ ], ['a.wav', 'a.wav'])
+ assert.deepEqual(resolved.audioUrls, ['a.wav', 'b.wav'])
+ assert.match(resolved.prompt, /小雨姐姐(音色参考音频2)/)
+ assert.match(resolved.prompt, /小雨(音色参考音频1)/)
+ assert.doesNotMatch(resolved.prompt, /#小雨/)
+ assert.match(resolved.prompt, /@小雨/)
+ assert.deepEqual(resolveVoiceMentions('@小雨 #01', [{ name: '小雨', voiceAudioUrl: 'a.wav' }]).audioUrls, [])
+ assert.equal(resolveVoiceMentions('#A B,C', [{ name: 'A B,C', voiceAudioUrl: 'a.wav' }]).hasVoiceReferences, true)
+})
+
+test('missing, ambiguous and unknown voice mentions fail rather than generating without the requested voice', () => {
+ assert.throws(() => resolveVoiceMentions('#小雨', [{ name: '小雨', voiceAudioUrl: null }]), /尚未上传/)
+ assert.throws(() => resolveVoiceMentions('#未知', []), /无效/)
+ assert.throws(() => resolveVoiceMentions('#小雨', [{ name: '小雨', voiceAudioUrl: 'a' }, { name: '小雨', voiceAudioUrl: 'b' }]), /同名/)
+})
+
+test('inline dialogue keeps speakers and delivery text in place, including repeated and overlapping names', () => {
+ const result = resolveVoiceMentions('0-3秒:@小雨抬头,#小雨轻声说:「早上好。」\n3-6秒:#小雨姐姐笑着说:「你好。」#小雨低声回应。', [
+ { name: '小雨', voiceAudioUrl: 'a.wav' }, { name: '小雨姐姐', voiceAudioUrl: 'b.wav' },
+ ])
+ assert.deepEqual(result.audioUrls, ['a.wav', 'b.wav'])
+ assert.equal(result.prompt, '0-3秒:@小雨抬头,小雨(音色参考音频1)轻声说:「早上好。」\n3-6秒:小雨姐姐(音色参考音频2)笑着说:「你好。」小雨(音色参考音频1)低声回应。')
+ assert.throws(() => resolveVoiceMentions('#小雨姐姐轻声说:「你好。」', [
+ { name: '小雨', voiceAudioUrl: 'a.wav' }, { name: '小雨姐姐', voiceAudioUrl: 'b.wav', bound: false },
+ ]), /未绑定/)
+})
+
+test('saved suffix references remain compatible and do not attach the same voice twice', () => {
+ const result = resolveVoiceMentions('使用#小雨音色,#小雨轻声说:「早上好。」', [{ name: '小雨', voiceAudioUrl: 'a.wav' }])
+ assert.deepEqual(result.audioUrls, ['a.wav'])
+ assert.equal(result.prompt, '使用小雨(音色参考音频1),小雨(音色参考音频1)轻声说:「早上好。」')
+ const namedWithSuffix = resolveVoiceMentions('#小雨音色说话', [
+ { name: '小雨', voiceAudioUrl: 'a.wav' }, { name: '小雨音色', voiceAudioUrl: 'b.wav' },
+ ])
+ assert.deepEqual(namedWithSuffix.audioUrls, ['b.wav'])
+})
+
+test('Seedance and MiniMax submissions carry local voice bytes and correct role bindings without a public URL', async () => {
+ const c = char('林小雨')
+ for (const provider of ['volcengine', 'minimax']) {
+ const response = await submit(shot(c), '0-3秒:@图片1林小雨抬头,#林小雨轻声说:「早上好。」', { config_id: configs[provider] })
+ assert.equal(response.status, 201, await response.clone().text())
+ const { data } = await response.json()
+ const task = await waitForSubmission(data.id)
+ const request = received.at(-1)
+ assert.match(request.content.find((c: any) => c.type === 'text').text, /林小雨.*音频1/)
+ const audio = request.content.filter((c: any) => c.type === 'audio_url')
+ assert.equal(audio.length, 1)
+ assert.equal(audio[0].role, 'reference_audio')
+ assert.equal(audio[0].audio_url.url, `data:audio/wav;base64,${wav(3).toString('base64')}`)
+ assert.deepEqual(JSON.parse(task.params!).referenceAudioUrls, ['static/uploads/3.wav'])
+ if (provider === 'volcengine') assert.equal(request.generate_audio, true)
+ }
+})
+
+test('image-only prompts never attach the character voice automatically', async () => {
+ const response = await submit(shot(char('小雨')), '@图片1小雨走路')
+ const { data } = await response.json()
+ await waitForSubmission(data.id)
+ assert.equal(received.at(-1).content.some((c: any) => c.type === 'audio_url'), false)
+})
+
+test('unbound, foreign and deleted roles are rejected before creating a generation task', async () => {
+ const otherDrama = db.insert(schema.dramas).values({ title: 'other', ...timestamps }).returning().get()
+ const foreign = char('外部角色', 'static/uploads/3.wav', otherDrama.id)
+ const deleted = char('已删除')
+ db.update(schema.characters).set({ deletedAt: ts }).where(eq(schema.characters.id, deleted.id)).run()
+ char('未绑定')
+ const before = db.select().from(schema.sysTask).all().length
+ for (const [id, prompt] of [[shot(), '#未绑定'], [shot(foreign), '#外部角色'], [shot(deleted), '#已删除']] as const) {
+ const response = await submit(id, prompt)
+ assert.equal(response.status, 400)
+ assert.match(await response.text(), /音色引用/)
+ }
+ assert.equal(db.select().from(schema.sysTask).all().length, before)
+})
+
+test('invalid voices, silent mode, missing visual references and provider count limits fail before submission', async () => {
+ const valid = char('有效')
+ const before = db.select().from(schema.sysTask).all().length
+ for (const [name, url, expected] of [
+ ['丢失', 'static/uploads/missing.wav', /不存在/],
+ ['太短', 'static/uploads/1.wav', /2–15/], ['太长', 'static/uploads/16.wav', /2–15/],
+ ] as const) {
+ const response = await submit(shot(char(name, url)), `#${name}`)
+ assert.equal(response.status, 400)
+ assert.match(await response.text(), expected)
+ }
+ for (const extra of [{ generate_audio: false }, { reference_image_urls: [] }, { config_id: configs.aliyun }]) {
+ const response = await submit(shot(valid), '#有效', extra)
+ assert.equal(response.status, 400)
+ }
+ const roles = [1, 2, 3, 4].map(n => char(`角色${n}`, `https://example.test/${n}.wav`))
+ const response = await submit(shot(...roles), roles.map(c => `#${c.name}`).join(' '))
+ assert.equal(response.status, 400)
+ assert.match(await response.text(), /音频≤3/)
+ assert.equal(db.select().from(schema.sysTask).all().length, before)
+})
+
+test('sample duration total and traversal are checked, and local M4A is converted and reused', async () => {
+ await fs.copyFile(path.join(root, 'static/uploads/9.wav'), path.join(root, 'static/uploads/another.wav'))
+ await assert.rejects(prepareReferenceAudio(['static/uploads/9.wav', 'static/uploads/another.wav'], 'volcengine'), /总时长/)
+ await assert.rejects(prepareReferenceAudio(['static/../outside.wav'], 'volcengine'), /无效/)
+ const m4a = path.join(root, 'static/uploads/sample.m4a')
+ await new Promise<void>((resolve, reject) => ffmpeg(path.join(root, 'static/uploads/3.wav')).audioCodec('aac')
+ .on('end', () => resolve()).on('error', reject).save(m4a))
+ const prepared = await prepareReferenceAudio(['static/uploads/sample.m4a'], 'volcengine')
+ assert.match(prepared[0], /^static\/voice-references\/[a-f0-9]+\.wav$/)
+ assert.deepEqual(await prepareReferenceAudio(['static/uploads/sample.m4a'], 'volcengine'), prepared)
+ assert.match((await resolveReferenceAudioUrls(prepared, 'volcengine'))[0], /^data:audio\/wav;base64,/)
+ process.env.PUBLIC_BASE_URL = 'https://assets.example.test/'
+ try {
+ assert.deepEqual(await resolveReferenceAudioUrls(prepared, 'aliyun'), [`https://assets.example.test/${prepared[0]}`])
+ } finally { delete process.env.PUBLIC_BASE_URL }
+})
+
+test('task logs omit inline audio bytes', t => {
+ const logs: string[] = []
+ t.mock.method(console, 'log', (...args: unknown[]) => logs.push(args.join(' ')))
+ logTaskPayload('test', 'audio', { content: [{ audio_url: { url: 'data:audio/wav;base64,private-audio' } }] })
+ assert.doesNotMatch(logs.join(''), /private-audio/)
+})
+
+test.after(async () => {
+ await new Promise<void>(resolve => server.close(() => resolve()))
+ db.$client.close()
+ await fs.rm(root, { recursive: true, force: true })
+})
diff --git a/frontend/app/components/CharacterVoice.vue b/frontend/app/components/CharacterVoice.vue
new file mode 100644
index 0000000..2260431
--- /dev/null
+++ b/frontend/app/components/CharacterVoice.vue
@@ -0,0 +1,118 @@
+<template>
+ <section class="character-voice" :aria-label="t('characterVoice.forCharacter', { name: character.name })" @click.stop @keydown.stop>
+ <div class="voice-heading">
+ <div class="voice-label"><AudioLines :size="14" /><span>{{ t('characterVoice.title') }}</span></div>
+ <button type="button" class="btn btn-sm" :disabled="busy" @click="fileInput?.click()">
+ <Loader2 v-if="busy" :size="12" class="animate-spin" />
+ <Upload v-else :size="12" />
+ {{ busy ? t('characterVoice.saving') : voiceUrl ? t('characterVoice.replace') : t('characterVoice.upload') }}
+ </button>
+ <input ref="fileInput" class="voice-file-input" type="file" accept=".mp3,.wav,.m4a,.aac,audio/mpeg,audio/wav,audio/mp4,audio/aac" :disabled="busy" :aria-label="t('characterVoice.upload')" @change="upload" />
+ </div>
+ <template v-if="voiceUrl">
+ <div class="voice-file-row">
+ <span class="voice-filename" :title="voiceName">{{ voiceName }}</span>
+ <button type="button" class="voice-remove" :disabled="busy" @click="remove">{{ t('characterVoice.remove') }}</button>
+ </div>
+ <audio :key="voiceUrl" :src="voiceUrl" controls preload="none" data-character-voice :aria-label="t('characterVoice.preview', { name: character.name })" @play="onPlay" @error="playbackError = true" />
+ <p v-if="playbackError" class="voice-error" role="alert">{{ t('characterVoice.playbackError') }}</p>
+ </template>
+ <p v-else class="voice-hint">{{ t('characterVoice.empty') }}</p>
+ <p class="voice-formats">{{ t('characterVoice.formats') }}</p>
+ </section>
+</template>
+
+<script setup lang="ts">
+import { AudioLines, Loader2, Upload } from 'lucide-vue-next'
+import { useI18n } from 'vue-i18n'
+import { toast } from 'vue-sonner'
+import { characterAPI, type CharacterVoice } from '~/composables/useApi'
+import { toastError } from '~/composables/useToast'
+
+const props = defineProps<{
+ character: {
+ id: number
+ name?: string
+ voice_audio_url?: string | null
+ voice_audio_name?: string | null
+ voiceAudioUrl?: string | null
+ voiceAudioName?: string | null
+ }
+}>()
+const emit = defineEmits<{ updated: [id: number, voice: CharacterVoice] }>()
+const { t } = useI18n()
+const fileInput = ref<HTMLInputElement | null>(null)
+// 卡片与详情弹窗共用同一个角色的上传锁。
+const busyIds = useState<number[]>('character-voice-busy', () => [])
+const busy = computed(() => busyIds.value.includes(props.character.id))
+const playbackError = ref(false)
+const voiceUrl = computed(() => {
+ const value = props.character.voice_audio_url ?? props.character.voiceAudioUrl ?? ''
+ return value && !value.startsWith('/') && !/^https?:\/\//i.test(value) ? `/${value}` : value
+})
+const voiceName = computed(() => props.character.voice_audio_name || props.character.voiceAudioName || t('characterVoice.sample'))
+watch(voiceUrl, () => { playbackError.value = false })
+
+async function upload(event: Event) {
+ const input = event.target as HTMLInputElement
+ const file = input.files?.[0]
+ input.value = ''
+ if (!file || busy.value) return
+ if (!/\.(mp3|wav|m4a|aac)$/i.test(file.name)) { toast.error(t('characterVoice.invalidFormat')); return }
+ if (!file.size) { toast.error(t('characterVoice.emptyFile')); return }
+ if (file.size > 20 * 1024 * 1024) { toast.error(t('characterVoice.tooLarge')); return }
+ const id = props.character.id
+ busyIds.value = [...busyIds.value, id]
+ try {
+ const voice = await characterAPI.uploadVoice(id, file)
+ emit('updated', id, voice)
+ toast.success(t('characterVoice.saved'))
+ } catch (error) {
+ toastError(error)
+ } finally {
+ busyIds.value = busyIds.value.filter(value => value !== id)
+ }
+}
+
+async function remove() {
+ if (busy.value) return
+ const id = props.character.id
+ busyIds.value = [...busyIds.value, id]
+ try {
+ const voice = await characterAPI.removeVoice(id)
+ emit('updated', id, voice)
+ toast.success(t('characterVoice.removed'))
+ } catch (error) {
+ toastError(error)
+ } finally {
+ busyIds.value = busyIds.value.filter(value => value !== id)
+ }
+}
+
+function onPlay(event: Event) {
+ playbackError.value = false
+ document.querySelectorAll<HTMLAudioElement>('audio[data-character-voice]').forEach(audio => {
+ if (audio !== event.target) audio.pause()
+ })
+}
+</script>
+
+<style scoped>
+.character-voice { min-width: 0; display: flex; flex-direction: column; gap: 8px; border-top: 1px solid var(--border); padding-top: 12px; margin-top: 4px; cursor: default; }
+.voice-heading, .voice-label, .voice-file-row { display: flex; align-items: center; gap: 8px; min-width: 0; }
+.voice-heading, .voice-file-row { justify-content: space-between; }
+.voice-label { color: var(--text-1); font-size: 12px; font-weight: 600; }
+.voice-label svg { color: var(--text-3); flex-shrink: 0; }
+.voice-heading .btn { flex-shrink: 0; }
+.voice-file-input { display: none; }
+.voice-filename { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-size: 12px; color: var(--text-2); }
+.voice-remove { flex-shrink: 0; background: none; border: none; padding: 4px; color: var(--text-3); font: inherit; font-size: 12px; cursor: pointer; transition: color 0.15s; }
+.voice-remove:hover { color: var(--action-danger); }
+.voice-remove:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; border-radius: 4px; }
+.voice-remove:disabled { opacity: 0.5; cursor: default; }
+.character-voice audio { display: block; width: 100%; min-width: 0; height: 36px; }
+.voice-hint, .voice-formats, .voice-error { margin: 0; font-size: 11px; line-height: 1.6; }
+.voice-hint { color: var(--text-2); }
+.voice-formats { color: var(--text-3); }
+.voice-error { color: var(--action-danger); }
+</style>
diff --git a/frontend/app/components/MentionTextarea.vue b/frontend/app/components/MentionTextarea.vue
index 4410ed3..3cb5cf0 100644
--- a/frontend/app/components/MentionTextarea.vue
+++ b/frontend/app/components/MentionTextarea.vue
@@ -33,12 +33,12 @@
<img v-if="opt.image" :src="opt.image" alt="" @error="$event.target.style.display = 'none'" />
<component v-else :is="groupIcon(opt.kind)" :size="12" :stroke-width="2" />
</span>
- <span class="mention-name">@{{ opt.label }}</span>
+ <span class="mention-name">{{ opt.trigger || '@' }}{{ opt.label }}</span>
<span class="mention-type">{{ opt.group }}</span>
</button>
</template>
</template>
- <div v-else class="mention-empty">{{ t('components.mention.noMatch') }}</div>
+ <div v-else class="mention-empty">{{ t(mention.trigger === '#' ? 'characterVoice.noMatch' : 'components.mention.noMatch') }}</div>
</div>
</Teleport>
</div>
@@ -46,17 +46,18 @@
<script setup>
import { ref, computed, watch, nextTick, onMounted, onBeforeUnmount } from 'vue'
-import { User, MapPin, Package } from 'lucide-vue-next'
+import { User, MapPin, Package, AudioLines } from 'lucide-vue-next'
import { useI18n } from 'vue-i18n'
+import { mentionToken, mentionEntries, activeMentionAt, highlightMentions, mentionDeletionRange } from '../utils/prompt-mentions.js'
const { t } = useI18n()
// 无图资产在下拉中显示分组图标兜底(场景=定位、道具=包裹、角色=人物)——按 kind code 判断,不依赖显示文案
-const groupIcon = (kind) => (kind === 'scene' ? MapPin : kind === 'prop' ? Package : User)
+const groupIcon = (kind) => (kind === 'voice' ? AudioLines : kind === 'scene' ? MapPin : kind === 'prop' ? Package : User)
const props = defineProps({
modelValue: { type: String, default: '' },
- // [{ label, value, kind, group, image? }] — value 为插入的 @引用名,kind('character'|'scene'|'prop')为逻辑值,group 为显示分组文案
+ // [{ label, value, kind, group, image?, trigger? }] — 默认 @图片引用,#角色音色引用
options: { type: Array, default: () => [] },
rows: { type: [Number, String], default: 4 },
placeholder: { type: String, default: '' },
@@ -68,7 +69,7 @@ const wrapEl = ref(null)
const taEl = ref(null)
const backdropEl = ref(null)
const text = ref(props.modelValue)
-const mention = ref({ open: false, start: 0, query: '', top: 0, left: 0 })
+const mention = ref({ open: false, start: 0, trigger: '@', query: '', top: 0, left: 0 })
const highlightIdx = ref(0)
const dropdownEl = ref(null)
@@ -86,7 +87,7 @@ const DROPDOWN_MAX_HEIGHT = 220
// 下拉 Teleport 到 body 后用 fixed 定位,任何外层滚动/窗口变化都会使其错位 → 直接关闭
function closeOnOuterScroll(e) {
- if (mention.value.open && e?.target !== taEl.value) closeMention()
+ if (mention.value.open && e?.target !== taEl.value && !dropdownEl.value?.contains(e?.target)) closeMention()
}
onMounted(() => {
window.addEventListener('scroll', closeOnOuterScroll, true)
@@ -98,80 +99,35 @@ onBeforeUnmount(() => {
})
watch(() => props.modelValue, (v) => {
- if (v !== text.value) text.value = v
- mention.value.open = false
+ if (v !== text.value) { text.value = v; mention.value.open = false }
})
// 可引用名(按长度降序,保证最长匹配优先)及其分组样式(kind code 判断)
-const mentionNames = computed(() => {
- const seen = new Map()
- for (const o of props.options) {
- if (o.value && !seen.has(o.value)) seen.set(o.value, o.kind === 'scene' ? 'scene' : 'role')
- }
- return [...seen.entries()].sort((a, b) => b[0].length - a[0].length)
-})
-
-function escapeHtml(s) {
- return s.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
-}
-function escapeRegExp(s) {
- return s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
-}
-
-const highlightRegex = computed(() => {
- if (!mentionNames.value.length) return null
- const alt = mentionNames.value.map(([name]) => escapeRegExp(name)).join('|')
- return new RegExp(`@(${alt})`, 'g')
-})
-
-// 镜像层高亮 HTML:@引用 包上分组色彩 span,其余文本原样
-const highlightedHtml = computed(() => {
- let html = escapeHtml(text.value)
- const re = highlightRegex.value
- if (re) {
- const groupOf = Object.fromEntries(mentionNames.value)
- html = html.replace(re, (m, name) =>
- `<span class="mention-token mention-token-${groupOf[name]}">@${escapeHtml(name)}</span>`)
- }
- if (html.endsWith('\n')) html += '\u200b'
- return html
-})
+const mentionNames = computed(() => mentionEntries(props.options))
+const highlightedHtml = computed(() => highlightMentions(text.value, props.options))
// 光标前最近一个 @token(@ 后不含空白),命中则处于引用输入态
function activeMention() {
const ta = taEl.value
if (!ta) return null
- const pos = ta.selectionStart
- const before = ta.value.slice(0, pos)
- const m = before.match(/@([^\s@]{0,20})$/)
- if (!m) return null
- return { start: pos - m[0].length, query: m[1] }
+ return activeMentionAt(ta.value, ta.selectionStart, props.options)
}
// 光标紧邻一个完整 @引用(用于整体删除)
function mentionBeforeCaret() {
const ta = taEl.value
- const pos = ta.selectionStart
- const before = ta.value.slice(0, pos)
- for (const [name] of mentionNames.value) {
- if (before.endsWith(`@${name}`)) return { start: pos - name.length - 1, end: pos }
- }
- return null
+ return mentionDeletionRange(ta.value, ta.selectionStart, props.options, true)
}
function mentionAfterCaret() {
const ta = taEl.value
- const pos = ta.selectionStart
- const after = ta.value.slice(pos)
- for (const [name] of mentionNames.value) {
- if (after.startsWith(`@${name}`)) return { start: pos, end: pos + name.length + 1 }
- }
- return null
+ return mentionDeletionRange(ta.value, ta.selectionStart, props.options, false)
}
const filteredOptions = computed(() => {
const q = mention.value.query.trim().toLowerCase()
- if (!q) return props.options
- return props.options.filter(o =>
+ const options = props.options.filter(o => (o.trigger || '@') === mention.value.trigger)
+ if (!q) return options
+ return options.filter(o =>
String(o.value).toLowerCase().includes(q) || String(o.label || '').toLowerCase().includes(q),
)
})
@@ -231,7 +187,7 @@ function updateMentionState() {
const active = activeMention()
if (!active) { mention.value.open = false; return }
// @后文本已精确等于某个完整引用名时不再弹列表(如删除引用尾部空格后光标落在完整引用上)
- if (mentionNames.value.some(([name]) => name === active.query)) {
+ if (mentionNames.value.some(([token]) => token === active.trigger + active.query)) {
mention.value.open = false
return
}
@@ -245,6 +201,7 @@ function updateMentionState() {
mention.value = {
open: true,
start: active.start,
+ trigger: active.trigger,
query: active.query,
top: above ? rect.top + caret.top - ta.scrollTop - 4 : belowTop,
left: rect.left + Math.min(caret.left, maxLeft),
@@ -285,6 +242,7 @@ function onScroll() {
}
function onKeydown(e) {
+ if (e.isComposing) return
const ta = taEl.value
// @引用整体删除:光标在完整引用后按退格 / 在完整引用前按 Delete,整段一次删除
if (ta && (e.key === 'Backspace' || e.key === 'Delete') && ta.selectionStart === ta.selectionEnd) {
@@ -321,7 +279,7 @@ function pick(opt) {
const pos = ta.selectionStart
const before = ta.value.slice(0, mention.value.start)
const after = ta.value.slice(pos)
- const insert = `@${opt.value} `
+ const insert = `${mentionToken(opt)}${opt.trigger === '#' ? '' : ' '}`
const next = before + insert + after
setText(next, before.length + insert.length)
}
@@ -385,6 +343,10 @@ function onBlur(e) {
color: var(--tag-success-text);
background: var(--success-bg);
}
+.mention-backdrop :deep(.mention-token-voice) {
+ color: var(--tag-success-text);
+ background: var(--success-bg);
+}
.mention-dropdown {
position: fixed;
z-index: var(--z-menu);
diff --git a/frontend/app/composables/useApi.ts b/frontend/app/composables/useApi.ts
index 600753a..e926fa3 100644
--- a/frontend/app/composables/useApi.ts
+++ b/frontend/app/composables/useApi.ts
@@ -65,6 +65,8 @@ export const storyboardAPI = {
}
export const characterAPI = {
+ uploadVoice: (id: number, file: File) => uploadReq<CharacterVoice>(`/characters/${id}/voice`, file),
+ removeVoice: (id: number) => api.del<CharacterVoice>(`/characters/${id}/voice`),
create: (data: any) => api.post('/characters', data),
update: (id: number, data: any) => api.put(`/characters/${id}`, data),
del: (id: number) => api.del(`/characters/${id}`),
@@ -73,6 +75,11 @@ export const characterAPI = {
batchImages: (ids: number[], episodeId: number, model?: string, configId?: number, textModel?: string, textConfigId?: number) => api.post('/characters/batch-generate-images', { character_ids: ids, episode_id: episodeId, model: model || undefined, config_id: configId || undefined, text_model: textModel || undefined, text_config_id: textConfigId || undefined }),
}
+export interface CharacterVoice {
+ voice_audio_url: string | null
+ voice_audio_name: string | null
+}
+
export const sceneAPI = {
create: (data: any) => api.post('/scenes', data),
update: (id: number, data: any) => api.put(`/scenes/${id}`, data),
diff --git a/frontend/app/locales/en.json b/frontend/app/locales/en.json
index b77d02e..94e0b06 100644
--- a/frontend/app/locales/en.json
+++ b/frontend/app/locales/en.json
@@ -3,6 +3,26 @@
"title": "Huobao Shorts",
"updateAvailable": "New version v{version} is available. Upgrade in Settings → About."
},
+ "characterVoice": {
+ "title": "Character voice",
+ "forCharacter": "Voice for {name}",
+ "upload": "Upload voice",
+ "replace": "Replace voice",
+ "remove": "Remove",
+ "saving": "Saving…",
+ "empty": "Upload a clear voice recording as this character’s voice sample.",
+ "formats": "MP3 / WAV / M4A / AAC · Up to 20MB",
+ "sample": "Voice sample",
+ "preview": "Preview voice for {name}",
+ "saved": "Character voice saved",
+ "removed": "Character voice removed",
+ "invalidFormat": "Choose an MP3, WAV, M4A or AAC audio file",
+ "tooLarge": "Audio must be no larger than 20MB",
+ "emptyFile": "The audio file is empty",
+ "playbackError": "Unable to play this audio. Check the file or upload it again.",
+ "referenceHint": "Use {'@'}Name for appearance and #Name at each spoken line, e.g. {'@'}Alex looks up; #Alex softly says hello. Use 2–5 second voice samples, up to 15 seconds total.",
+ "noMatch": "No matching voice. Upload a character voice in Assets and add the character to this shot."
+ },
"common": {
"confirm": "Confirm",
"cancel": "Cancel",
@@ -673,7 +693,7 @@
"regenPrompt": "Regenerate",
"aiGenerate": "AI Generate",
"videoPromptCopy": "Generated from this shot's visual description (including dialogue/narration) and atmosphere",
- "videoPromptPlaceholder": "Reference assets with {'@'}character / {'@'}scene, describe motion and camera per 3-second line; or click AI Generate for the prompt Agent…",
+ "videoPromptPlaceholder": "Use {'@'}Name for images and #Name for voices; describe the dialogue, speaker and camera movement…",
"breaking": "Breaking the storyboard...",
"emptyTitle": "Start Storyboard Breakdown",
"emptyDesc": "Break the script into shots based on the script, characters and scenes, producing shot descriptions and bindings.",
@@ -766,7 +786,7 @@
"aspectRatio": "Aspect Ratio"
},
"inspector": {
- "videoPromptPlaceholder": "Reference assets with {'@'}character / {'@'}scene — auto-mapped to reference images on generation; then describe motion and camera per time segment…",
+ "videoPromptPlaceholder": "Use {'@'}Name for images and #Name for voices; describe the dialogue, speaker and camera movement…",
"noRefs": "No reference assets bound to this shot",
"refMedia": "Reference images / videos / audio",
"imageN": "Image {n}",
diff --git a/frontend/app/locales/ja.json b/frontend/app/locales/ja.json
index abfd415..b802f35 100644
--- a/frontend/app/locales/ja.json
+++ b/frontend/app/locales/ja.json
@@ -3,6 +3,26 @@
"title": "火宝ショートドラマ",
"updateAvailable": "新バージョン v{version} があります。「設定 → 更新について」でアップグレードできます"
},
+ "characterVoice": {
+ "title": "キャラクターの声",
+ "forCharacter": "{name}の声",
+ "upload": "音声をアップロード",
+ "replace": "音声を差し替え",
+ "remove": "解除",
+ "saving": "保存中…",
+ "empty": "キャラクターの声のサンプルとして、明瞭な音声をアップロードします。",
+ "formats": "MP3 / WAV / M4A / AAC · 最大20MB",
+ "sample": "音声サンプル",
+ "preview": "{name}の声を試聴",
+ "saved": "キャラクターの声を保存しました",
+ "removed": "キャラクターの声を解除しました",
+ "invalidFormat": "MP3、WAV、M4A、AACの音声ファイルを選択してください",
+ "tooLarge": "音声ファイルは20MB以下にしてください",
+ "emptyFile": "音声ファイルが空です",
+ "playbackError": "音声を再生できません。ファイルを確認するか、再アップロードしてください。",
+ "referenceHint": "{'@'}名前で姿、セリフの #名前で音声を参照。例:{'@'}太郎が顔を上げ、#太郎が優しく話す。音声は各 2〜5 秒、合計 15 秒以内を推奨。",
+ "noMatch": "該当する音声がありません。アセットで音声をアップロードし、このショットにキャラクターを追加してください。"
+ },
"common": {
"confirm": "確認",
"cancel": "キャンセル",
@@ -673,7 +693,7 @@
"regenPrompt": "再生成",
"aiGenerate": "AI で生成",
"videoPromptCopy": "この分镜の画面説明(セリフ/ナレーションを含む)と雰囲気から生成",
- "videoPromptPlaceholder": "{'@'}キャラクター名 / {'@'}シーン名 で参照素材を指定し、3 秒ごとに改行して画面の動きとカメラを記述。AI で生成してプロンプト Agent に任せることもできます…",
+ "videoPromptPlaceholder": "{'@'}名前で画像、#名前で音声を参照。セリフ、話者、カメラの動きを記述…",
"breaking": "分镜を分解中...",
"emptyTitle": "分镜分解を開始",
"emptyDesc": "脚本、キャラクター、シーンをもとにショットへ分解し、分镜説明と紐付け情報を生成します。",
@@ -766,7 +786,7 @@
"aspectRatio": "アスペクト比"
},
"inspector": {
- "videoPromptPlaceholder": "{'@'}キャラクター名 / {'@'}シーン名 で参照素材を指定(生成時に参照画像へ自動マッピング)。時間帯ごとに画面の動きとカメラを記述…",
+ "videoPromptPlaceholder": "{'@'}名前で画像、#名前で音声を参照。セリフ、話者、カメラの動きを記述…",
"noRefs": "この分镜には参照素材が紐付けられていません",
"refMedia": "参照画像 / 動画 / 音声",
"imageN": "画像 {n}",
diff --git a/frontend/app/locales/ko.json b/frontend/app/locales/ko.json
index c46a630..0f80361 100644
--- a/frontend/app/locales/ko.json
+++ b/frontend/app/locales/ko.json
@@ -3,6 +3,26 @@
"title": "화바오 숏드라마",
"updateAvailable": "새 버전 v{version}을 사용할 수 있습니다. 「설정 → 업데이트 정보」에서 업그레이드하세요"
},
+ "characterVoice": {
+ "title": "캐릭터 음색",
+ "forCharacter": "{name}의 음색",
+ "upload": "음색 업로드",
+ "replace": "음색 교체",
+ "remove": "제거",
+ "saving": "저장 중…",
+ "empty": "캐릭터 음색 샘플로 사용할 선명한 음성 녹음을 업로드하세요.",
+ "formats": "MP3 / WAV / M4A / AAC · 최대 20MB",
+ "sample": "음색 샘플",
+ "preview": "{name}의 음색 미리 듣기",
+ "saved": "캐릭터 음색이 저장되었습니다",
+ "removed": "캐릭터 음색이 제거되었습니다",
+ "invalidFormat": "MP3, WAV, M4A 또는 AAC 오디오 파일을 선택하세요",
+ "tooLarge": "오디오 파일은 20MB 이하여야 합니다",
+ "emptyFile": "오디오 파일이 비어 있습니다",
+ "playbackError": "오디오를 재생할 수 없습니다. 파일을 확인하거나 다시 업로드하세요.",
+ "referenceHint": "{'@'}이름으로 외형, 대사에서 #이름으로 음색을 참조하세요. 예: {'@'}민수가 고개를 들고 #민수가 부드럽게 말한다. 샘플당 2~5초, 총 15초 이하를 권장합니다.",
+ "noMatch": "일치하는 음색이 없습니다. 에셋에서 음색을 업로드하고 현재 샷에 캐릭터를 추가하세요."
+ },
"common": {
"confirm": "확인",
"cancel": "취소",
@@ -673,7 +693,7 @@
"regenPrompt": "다시 생성",
"aiGenerate": "AI로 생성",
"videoPromptCopy": "이 스토리보드의 화면 설명(대사/나레이션 포함)과 분위기를 바탕으로 생성",
- "videoPromptPlaceholder": "{'@'}캐릭터명 / {'@'}장멸명 으로 참조 소재를 지정하고, 3초마다 줄 바꿈하여 화면의 움직임과 침라를 기술. AI로 생성해 프롬프트 Agent에 맡길 수도 있습니다…",
+ "videoPromptPlaceholder": "{'@'}이름으로 이미지, #이름으로 음색을 참조하고 대사, 화자, 카메라 움직임을 설명하세요…",
"breaking": "스토리보드 분해 중...",
"emptyTitle": "스토리보드 분해 시작",
"emptyDesc": "대본, 캐릭터, 장면을 바탕으로 샷으로 분해하여 스토리보드 설명과 연결 정보를 생성합니다.",
@@ -766,7 +786,7 @@
"aspectRatio": "화면 비율"
},
"inspector": {
- "videoPromptPlaceholder": "{'@'}캐릭터명 / {'@'}장멸명 으로 참조 소재를 지정(생성 시 참조 이미지로 자동 매핑). 시간대별로 화면의 움직임과 침라를 기술…",
+ "videoPromptPlaceholder": "{'@'}이름으로 이미지, #이름으로 음색을 참조하고 대사, 화자, 카메라 움직임을 설명하세요…",
"noRefs": "이 스토리보드에는 참조 소재가 연결되어 있지 않습니다",
"refMedia": "참조 이미지 / 영상 / 오디오",
"imageN": "이미지 {n}",
diff --git a/frontend/app/locales/zh.json b/frontend/app/locales/zh.json
index 8fb5b02..767e42e 100644
--- a/frontend/app/locales/zh.json
+++ b/frontend/app/locales/zh.json
@@ -3,6 +3,26 @@
"title": "火宝短剧",
"updateAvailable": "发现新版本 v{version},可在「设置 → 关于更新」中升级"
},
+ "characterVoice": {
+ "title": "角色音色",
+ "forCharacter": "{name}的角色音色",
+ "upload": "上传音色",
+ "replace": "替换音色",
+ "remove": "移除",
+ "saving": "保存中…",
+ "empty": "上传一段清晰的人声,作为角色的音色样本。",
+ "formats": "MP3 / WAV / M4A / AAC · 最大 20MB",
+ "sample": "音色样本",
+ "preview": "试听{name}的音色",
+ "saved": "角色音色已保存",
+ "removed": "已移除角色音色",
+ "invalidFormat": "请选择 MP3、WAV、M4A 或 AAC 音频文件",
+ "tooLarge": "音频文件不能超过 20MB",
+ "emptyFile": "音频文件不能为空",
+ "playbackError": "音频无法播放,请检查文件或重新上传。",
+ "referenceHint": "用 {'@'}角色名引用形象,在对白处用 #角色名引用音色,例如:{'@'}林小雨抬头,#林小雨轻声说。音色建议每段 2–5 秒,总计不超过 15 秒。",
+ "noMatch": "未找到音色,请先在资产制作中上传角色音色,并将角色绑定到当前分镜。"
+ },
"common": {
"confirm": "确认",
"cancel": "取消",
@@ -673,7 +693,7 @@
"regenPrompt": "重新生成",
"aiGenerate": "AI 生成",
"videoPromptCopy": "根据当前分镜的画面描述(含台词/旁白)与氛围生成",
- "videoPromptPlaceholder": "用 {'@'}角色名 / {'@'}场景名 引用参考素材,按 3 秒一段换行描述画面运动与镜头;也可点 AI 生成由提示词 Agent 自动创作…",
+ "videoPromptPlaceholder": "用 {'@'}角色名引用图片,用 #角色名引用声音;描述台词、说话角色与镜头运动…",
"breaking": "正在拆分分镜...",
"emptyTitle": "开始拆分分镜",
"emptyDesc": "根据剧本、角色和场景拆分镜头,生成分镜描述和绑定信息。",
@@ -766,7 +786,7 @@
"aspectRatio": "画面比例"
},
"inspector": {
- "videoPromptPlaceholder": "用 {'@'}角色名 / {'@'}场景名 引用参考素材,生成时自动映射为参考图片;再按时间段描述画面运动与镜头…",
+ "videoPromptPlaceholder": "用 {'@'}角色名引用图片,用 #角色名引用声音;描述台词、说话角色与镜头运动…",
"noRefs": "当前分镜未绑定参考素材",
"refMedia": "参考图片 / 视频 / 音频",
"imageN": "图片 {n}",
diff --git a/frontend/app/utils/prompt-mentions.js b/frontend/app/utils/prompt-mentions.js
new file mode 100644
index 0000000..9a9e0f8
--- /dev/null
+++ b/frontend/app/utils/prompt-mentions.js
@@ -0,0 +1,53 @@
+export const mentionToken = option => `${option.trigger || '@'}${option.value}`
+
+export function mentionEntries(options) {
+ const seen = new Map()
+ for (const option of options) {
+ const token = mentionToken(option)
+ if (option.value && !seen.has(token)) {
+ seen.set(token, option.kind === 'voice' ? 'voice' : option.kind === 'scene' ? 'scene' : 'role')
+ }
+ }
+ // Legacy saved voice tokens remain highlighted/deletable; new insertions use #Name.
+ for (const option of options) {
+ for (const alias of option.aliases || []) {
+ const token = `${option.trigger || '@'}${alias}`
+ if (!seen.has(token)) seen.set(token, option.kind === 'voice' ? 'voice' : 'role')
+ }
+ }
+ return [...seen.entries()].sort((a, b) => b[0].length - a[0].length)
+}
+
+export function activeMentionAt(text, position, options) {
+ const match = text.slice(0, position).match(/([@#])([^\s@#]{0,100})$/)
+ // Chinese dialogue follows a token without spaces: #林小雨轻声说. Do not reopen
+ // the dropdown while typing the dialogue, but allow searching for a longer name.
+ if (match && options && mentionEntries(options).some(([token]) => match[0].startsWith(token)) &&
+ !options.some(option => (option.trigger || '@') === match[1] && String(option.value).toLowerCase().includes(match[2].toLowerCase()))) return null
+ return match ? { start: position - match[0].length, trigger: match[1], query: match[2] } : null
+}
+
+const escapeHtml = text => text.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;')
+const escapeRegExp = text => text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+
+export function highlightMentions(text, options) {
+ const entries = mentionEntries(options)
+ if (!entries.length) return escapeHtml(text) + (text.endsWith('\n') ? '\u200b' : '')
+ const pattern = new RegExp(entries.map(([token]) => escapeRegExp(token)).join('|'), 'g')
+ const styles = new Map(entries)
+ let html = '', start = 0
+ for (const match of text.matchAll(pattern)) {
+ html += escapeHtml(text.slice(start, match.index))
+ html += `<span class="mention-token mention-token-${styles.get(match[0])}">${escapeHtml(match[0])}</span>`
+ start = match.index + match[0].length
+ }
+ return html + escapeHtml(text.slice(start)) + (text.endsWith('\n') ? '\u200b' : '')
+}
+
+export function mentionDeletionRange(text, position, options, backwards) {
+ for (const [token] of mentionEntries(options)) {
+ if (backwards && text.slice(0, position).endsWith(token)) return { start: position - token.length, end: position }
+ if (!backwards && text.slice(position).startsWith(token)) return { start: position, end: position + token.length }
+ }
+ return null
+}
diff --git a/frontend/app/views/drama/detail.vue b/frontend/app/views/drama/detail.vue
index 1ec34e1..d8f3afc 100644
--- a/frontend/app/views/drama/detail.vue
+++ b/frontend/app/views/drama/detail.vue
@@ -217,8 +217,8 @@
tabindex="0"
role="button"
@click="openEdit(m)"
- @keydown.enter.prevent="openEdit(m)"
- @keydown.space.prevent="openEdit(m)"
+ @keydown.enter.self.prevent="openEdit(m)"
+ @keydown.space.self.prevent="openEdit(m)"
>
<div class="character-asset-main">
<div class="character-asset-overview">
@@ -257,6 +257,7 @@
<span class="afp-label">{{ t('episode.asset.finalPromptTurnaround') }}</span>
<span :class="['afp-text', !m.finalPrompt && 'dim']">{{ m.finalPrompt || t('episode.asset.finalPromptAutoTurnaround') }}</span>
</div>
+ <CharacterVoice :character="m" @updated="updateCharacterVoice" />
</div>
</article>
</div>
@@ -383,6 +384,7 @@
<strong>{{ editTarget.kindKey === 'character' ? (editTarget.role || t('common.role')) : editTarget.kindKey === 'prop' ? (editTarget.type || t('common.prop')) : (editTarget.time || t('episode.asset.noTime')) }}</strong>
</div>
</div>
+ <CharacterVoice v-if="editTarget.kindKey === 'character'" :key="editTarget.id" :character="editTarget" @updated="updateCharacterVoice" />
</aside>
<!-- 右侧:编辑信息 -->
@@ -845,6 +847,12 @@ function fmtDate(s) {
const editDialog = ref(false)
const editSaving = ref(false)
const editTarget = ref(null)
+function updateCharacterVoice(id, voice) {
+ const patch = { ...voice, voiceAudioUrl: voice.voice_audio_url, voiceAudioName: voice.voice_audio_name }
+ const character = drama.value?.characters?.find(item => item.id === id)
+ if (character) Object.assign(character, patch)
+ if (editTarget.value?.kindKey === 'character' && editTarget.value.id === id) Object.assign(editTarget.value, patch)
+}
const editDraft = reactive({})
function openEdit(m) {
diff --git a/frontend/app/views/drama/episode.vue b/frontend/app/views/drama/episode.vue
index 5d405bc..5334fca 100644
--- a/frontend/app/views/drama/episode.vue
+++ b/frontend/app/views/drama/episode.vue
@@ -313,8 +313,8 @@
tabindex="0"
role="button"
@click="openAssetDetail('character', c)"
- @keydown.enter.prevent="openAssetDetail('character', c)"
- @keydown.space.prevent="openAssetDetail('character', c)"
+ @keydown.enter.self.prevent="openAssetDetail('character', c)"
+ @keydown.space.self.prevent="openAssetDetail('character', c)"
>
<button class="asset-del-btn" :title="t('episode.asset.delChar')" @click.stop="askDeleteAsset('character', c)"><X :size="11" /></button>
<div class="character-asset-main">
@@ -361,6 +361,7 @@
<span class="afp-label">{{ t('episode.asset.finalPromptTurnaround') }}</span>
<span :class="['afp-text', !(c.final_prompt || c.finalPrompt) && 'dim']">{{ c.final_prompt || c.finalPrompt || t('episode.asset.finalPromptAutoTurnaround') }}</span>
</div>
+ <CharacterVoice :character="c" @updated="updateCharacterVoice" />
</div>
</article>
</div>
@@ -689,6 +690,7 @@
:placeholder="t('episode.inspector.videoPromptPlaceholder')"
@commit="v => updateField(selectedSb, 'video_prompt', v)"
/>
+ <p class="voice-reference-hint">{{ t('characterVoice.referenceHint') }}</p>
</section>
</div>
</div>
@@ -1150,6 +1152,7 @@
<strong>{{ assetDetail.type === 'character' ? (assetDetail.item.role || t('common.role')) : assetDetail.type === 'prop' ? (assetDetail.item.type || t('common.prop')) : (assetDetail.item.time || t('episode.asset.noTime')) }}</strong>
</div>
</div>
+ <CharacterVoice v-if="assetDetail.type === 'character'" :key="assetDetail.item.id" :character="assetDetail.item" @updated="updateCharacterVoice" />
</aside>
<section class="asset-detail-editor-panel">
@@ -1651,6 +1654,12 @@ async function toggleExportDone() {
}
}
const assetDetail = ref({ open: false, type: '', item: null })
+function updateCharacterVoice(id, voice) {
+ const patch = { ...voice, voiceAudioUrl: voice.voice_audio_url, voiceAudioName: voice.voice_audio_name }
+ const character = chars.value.find(item => item.id === id)
+ if (character) Object.assign(character, patch)
+ if (assetDetail.value.type === 'character' && assetDetail.value.item?.id === id) Object.assign(assetDetail.value.item, patch)
+}
const assetDetailDraft = ref({ appearance: '', styling: '', prompt: '', lighting: '', description: '' })
// 最终提示词手动编辑:dirty 时才随保存提交,避免无修改保存误清空 Agent 生成的提示词
const assetPromptDraft = ref('')
@@ -3239,6 +3248,14 @@ const mentionOptions = computed(() => {
group: t('common.role'),
image: thumbOf(assetImageSrc(c)),
})),
+ ...getStoryboardCharacters(sb).filter(c => c.voice_audio_url || c.voiceAudioUrl).map(c => ({
+ label: c.name,
+ value: c.name,
+ aliases: [`${c.name}音色`],
+ trigger: '#',
+ kind: 'voice',
+ group: t('characterVoice.title'),
+ })),
...(scene ? [{
label: `${scene.location} · ${scene.time || t('episode.asset.noTime')}`,
value: scene.location,
@@ -3284,7 +3301,7 @@ function resolveVideoPromptRefs(sb) {
const map = getShotReferenceIndexMap(sb)
const names = Object.keys(map).sort((a, b) => b.length - a.length)
if (!names.length) return prompt
- return prompt.replace(/@([^\s@]+)/g, (m, raw) => {
+ return prompt.replace(/@([^\s@#]+)/g, (m, raw) => {
for (const name of names) {
if (raw.startsWith(name)) {
return `@图片${map[name]}${name}${raw.slice(name.length)}`
@@ -3473,6 +3490,12 @@ onMounted(() => setTimeout(() => autoTour('episode', EPISODE_TOUR, t), 900))
</script>
<style scoped>
+.voice-reference-hint {
+ margin: 8px 0 0;
+ font-size: 11px;
+ line-height: 1.6;
+ color: var(--text-3);
+}
/* ===== Studio Layout ===== */
.studio {
display: flex;
diff --git a/frontend/tests/prompt-mentions.test.mjs b/frontend/tests/prompt-mentions.test.mjs
new file mode 100644
index 0000000..20643e8
--- /dev/null
+++ b/frontend/tests/prompt-mentions.test.mjs
@@ -0,0 +1,46 @@
+import { test } from 'node:test'
+import assert from 'node:assert/strict'
+import { activeMentionAt, mentionToken, highlightMentions, mentionDeletionRange } from '../app/utils/prompt-mentions.js'
+
+const options = [
+ { value: '小雨', kind: 'character' },
+ { value: '小雨', aliases: ['小雨音色'], trigger: '#', kind: 'voice' },
+ { value: '小雨姐姐', aliases: ['小雨姐姐音色'], trigger: '#', kind: 'voice' },
+ { value: 'A&B<', trigger: '#', kind: 'voice' },
+]
+
+test('image and voice triggers stay independent, including adjacent references', () => {
+ assert.deepEqual(activeMentionAt('@小雨#小', 5), { start: 3, trigger: '#', query: '小' })
+ assert.deepEqual(activeMentionAt('场景 @', 4), { start: 3, trigger: '@', query: '' })
+ assert.equal(activeMentionAt('#小雨音色 台词', 8), null)
+ assert.equal(mentionToken(options[0]), '@小雨')
+ assert.equal(mentionToken(options[1]), '#小雨')
+})
+
+test('highlighting preserves literal text, safely escapes HTML, and prefers complete names', () => {
+ const html = highlightMentions('@小雨抬头,#小雨姐姐轻声说,#A&B<回应 <script>\n', options)
+ assert.match(html, /mention-token-role">@小雨<\/span>/)
+ assert.match(html, /mention-token-voice">#小雨姐姐<\/span>轻声说/)
+ assert.match(html, /mention-token-voice">#A&amp;B&lt;<\/span>回应/)
+ assert.match(html, /&lt;script&gt;\n\u200b$/)
+ assert.equal(highlightMentions('#未知音色', options), '#未知音色')
+})
+
+test('Backspace and Delete remove whole voice tokens without consuming surrounding text', () => {
+ const text = '@小雨#小雨轻声说话'
+ assert.deepEqual(mentionDeletionRange(text, 6, options, true), { start: 3, end: 6 })
+ assert.deepEqual(mentionDeletionRange(text, 3, options, false), { start: 3, end: 6 })
+ assert.equal(mentionDeletionRange(text, 5, options, true), null)
+ assert.deepEqual(mentionDeletionRange(text, 3, options, true), { start: 0, end: 3 })
+})
+
+test('typing dialogue immediately after a voice name keeps the menu closed; longer names can still be searched', () => {
+ const speech = '#小雨轻声说'
+ assert.equal(activeMentionAt(speech, speech.length, options), null)
+ assert.deepEqual(activeMentionAt('#小雨姐', 4, options), { start: 0, trigger: '#', query: '小雨姐' })
+})
+
+test('previously saved suffix tokens are highlighted and deleted as one mention', () => {
+ assert.match(highlightMentions('#小雨音色说话', options), /mention-token-voice">#小雨音色<\/span>说话/)
+ assert.deepEqual(mentionDeletionRange('#小雨音色说话', 5, options, true), { start: 0, end: 5 })
+})
diff --git a/frontend/tests/remove-audio-voice-structure.test.mjs b/frontend/tests/remove-audio-voice-structure.test.mjs
index d5a092c..a95b419 100644
--- a/frontend/tests/remove-audio-voice-structure.test.mjs
+++ b/frontend/tests/remove-audio-voice-structure.test.mjs
@@ -5,8 +5,8 @@ import assert from 'node:assert/strict'
const appRoot = new URL('../app/', import.meta.url)
const readApp = (path) => readFileSync(new URL(path, appRoot), 'utf8')
-const episodePage = readApp('pages/drama/[id]/episode/[episodeNumber].vue')
-const dramaPage = readApp('pages/drama/[id]/index.vue')
+const episodePage = readApp('views/drama/episode.vue')
+const dramaPage = readApp('views/drama/detail.vue')
const settingsPage = readApp('pages/settings.vue')
const useApi = readApp('composables/useApi.ts')
@@ -19,14 +19,12 @@ test('frontend API client no longer exposes TTS or voice endpoints', () => {
assert.doesNotMatch(useApi, /generate-tts/)
})
-test('episode workbench removes all role voice assignment controls', () => {
+test('uploaded character voices do not restore AI voice assignment or TTS controls', () => {
assert.doesNotMatch(episodePage, /voice_assigner/)
assert.doesNotMatch(episodePage, /AI 匹配声音/)
assert.doesNotMatch(episodePage, /批量试听/)
- assert.doesNotMatch(episodePage, /角色音色/)
assert.doesNotMatch(episodePage, /已配音色/)
assert.doesNotMatch(episodePage, /待音色/)
- assert.doesNotMatch(episodePage, /asset-detail-voice-panel/)
assert.doesNotMatch(episodePage, /updateCharVoice/)
assert.doesNotMatch(episodePage, /previewVoiceSample/)
assert.doesNotMatch(episodePage, /voiceProfiles/)