Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
docs: fix GitHub Pages API publishing flow
游雁3b608cf9e158
diff --git a/.github/workflows/update-api-docs.yml b/.github/workflows/update-api-docs.ymlindex 2b32aaf4b..817526021 100644--- a/.github/workflows/update-api-docs.yml+++ b/.github/workflows/update-api-docs.yml@@ -5,6 +5,9 @@ on:branches: [main]paths:- 'funasr/**/*.py'+ - 'scripts/gen_api_docs.py'+ - 'training.html'+ - '.github/workflows/update-api-docs.yml'jobs:build-api-docs:diff --git a/gh-pages-output/api.html b/gh-pages-output/api.htmlindex dc7bcbffb..2653a9b03 100644--- a/gh-pages-output/api.html+++ b/gh-pages-output/api.html@@ -48,7 +48,7 @@ h2{font-size:1.1rem;margin:8px 0}h2 code{font-family:'JetBrains Mono',monospace;</style></head><body><nav class="nav"><div class="container"><a href="index.html" class="nav-logo">FunASR</a>-<div class="nav-links"><a href="index.html">Home</a><a href="tutorial.html">Tutorial</a><a href="model-registration.html">Develop</a><a href="api.html" class="active">API</a></div>+<div class="nav-links"><a href="index.html">Home</a><a href="tutorial.html">Tutorial</a><a href="training.html">Training</a><a href="model-registration.html">Develop</a><a href="api.html" class="active">API</a></div><a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a></div></nav><div class="api-layout">diff --git a/gh-pages-output/training.html b/gh-pages-output/training.htmlnew file mode 100644index 000000000..86d44b152--- /dev/null+++ b/gh-pages-output/training.html@@ -0,0 +1,348 @@+<!DOCTYPE html>+<html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0">+<title>FunASR Training Guide</title>+<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700;800&family=JetBrains+Mono:wght@400;500&display=swap" rel="stylesheet">+<link rel="stylesheet" href="style.css">+<style>+.content{padding:48px 0 80px}.container.narrow{max-width:820px;margin:0 auto;padding:0 24px}+h1{font-size:2.2rem;font-weight:800;letter-spacing:-0.5px;margin-bottom:8px}+h1+p{color:var(--muted);margin-bottom:32px;font-size:1.05rem}+h2{font-size:1.4rem;font-weight:700;margin:56px 0 16px;padding-bottom:10px;border-bottom:2px solid var(--line)}+h3{font-size:1.1rem;font-weight:600;margin:32px 0 12px}+h4{font-size:0.95rem;font-weight:600;margin:24px 0 8px;color:var(--ink-soft)}+p{margin-bottom:12px;line-height:1.75;font-size:0.92rem}+ul,ol{margin:12px 0 12px 24px}li{margin-bottom:8px;line-height:1.6;font-size:0.9rem}+pre{background:var(--code-bg,#101828);color:#e2e8f0;border-radius:8px;padding:18px 22px;overflow-x:auto;margin:14px 0 22px;font-size:0.82rem;line-height:1.55}+code{font-family:'JetBrains Mono',monospace}+p code,li code,td code{background:#f1f5f9;padding:2px 6px;border-radius:4px;font-size:0.82rem;color:#b91c1c}+.note{background:#eff6ff;border-left:4px solid #2563eb;padding:14px 18px;border-radius:0 8px 8px 0;margin:16px 0;font-size:0.88rem}+.warning{background:#fffbeb;border-left:4px solid #f59e0b;padding:14px 18px;border-radius:0 8px 8px 0;margin:16px 0;font-size:0.88rem}+.tip{background:#f0fdf4;border-left:4px solid #059669;padding:14px 18px;border-radius:0 8px 8px 0;margin:16px 0;font-size:0.88rem}+table{width:100%;border-collapse:collapse;margin:16px 0;font-size:0.88rem}+th,td{padding:10px 14px;text-align:left;border-bottom:1px solid var(--line)}+th{background:#f8fafc;font-weight:600;font-size:0.78rem;text-transform:uppercase;letter-spacing:0.3px;color:var(--muted)}+.toc-grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(180px,1fr));gap:8px;margin:20px 0 40px}+.toc-grid a{display:block;padding:10px 14px;background:var(--page,#f7f9fc);border:1px solid var(--line);border-radius:8px;text-decoration:none;color:var(--ink-soft);font-size:0.82rem;font-weight:500;transition:all 0.15s}+.toc-grid a:hover{border-color:#2563eb;color:#2563eb}+.output-box{background:#f8fafc;border:1px solid var(--line);border-radius:8px;padding:14px 18px;margin:12px 0;font-family:'JetBrains Mono',monospace;font-size:0.76rem;line-height:1.6;overflow-x:auto}+</style>+</head><body>+<nav class="nav"><div class="container">+<a href="index.html" class="nav-logo">FunASR</a>+<div class="nav-links">+<a href="index.html">Home</a>+<a href="tutorial.html">Tutorial</a>+<a href="training.html" class="active">Training</a>+<a href="model-registration.html">Develop</a>+<a href="api.html">API</a>+</div>+<a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a>+</div></nav>++<div class="content"><div class="container narrow">+<h1>Training & Fine-tuning</h1>+<p>Fine-tune pretrained models on your own data using FunASR's training framework.</p>++<div class="toc-grid">+<a href="#overview">Overview</a>+<a href="#data">Data Preparation</a>+<a href="#paraformer">Fine-tune Paraformer</a>+<a href="#sensevoice">Fine-tune SenseVoice</a>+<a href="#nano">Fine-tune Fun-ASR-Nano</a>+<a href="#params">Parameter Reference</a>+<a href="#multi-gpu">Multi-GPU Training</a>+<a href="#deepspeed">DeepSpeed</a>+<a href="#monitor">Monitoring</a>+<a href="#inference-after">Use Fine-tuned Model</a>+<a href="#tips">Tips & Troubleshooting</a>+</div>++<!-- Overview -->+<h2 id="overview">Overview</h2>+<p>FunASR's training framework supports:</p>+<ul>+<li><strong>Fine-tuning</strong> any pretrained model on custom domain data</li>+<li><strong>Multi-GPU</strong> training with PyTorch DDP (single/multi-node)</li>+<li><strong>DeepSpeed</strong> ZeRO Stage 1/2/3 for large model training</li>+<li><strong>Dynamic batching</strong> by token count or example count</li>+<li><strong>Checkpoint averaging</strong> for best performance</li>+<li><strong>Resume training</strong> from interruption</li>+</ul>++<p>The training entry point is <code>funasr-train-ds</code> (or <code>funasr/bin/train_ds.py</code>), launched via <code>torchrun</code> for distributed training.</p>++<!-- Data -->+<h2 id="data">Data Preparation</h2>++<h3>Standard Format (Paraformer, SenseVoice)</h3>+<p>Training data uses JSONL format — one JSON object per line:</p>+<pre>{"key": "utt001", "source": "/path/to/audio.wav", "source_len": 90, "target": "这是转写文本", "target_len": 6}+{"key": "utt002", "source": "/path/to/audio2.wav", "source_len": 150, "target": "hello world", "target_len": 2}</pre>++<table>+<tr><th>Field</th><th>Type</th><th>Description</th></tr>+<tr><td><code>key</code></td><td>str</td><td>Unique utterance ID</td></tr>+<tr><td><code>source</code></td><td>str</td><td>Audio file path (local path or URL)</td></tr>+<tr><td><code>source_len</code></td><td>int</td><td>Audio length in fbank frames (1 frame = 10ms)</td></tr>+<tr><td><code>target</code></td><td>str</td><td>Transcription text</td></tr>+<tr><td><code>target_len</code></td><td>int</td><td>Number of text tokens</td></tr>+</table>++<h4>Generate from wav.scp + text.txt</h4>+<p>If you have Kaldi-style data files, convert them:</p>+<pre># train_wav.scp (tab-separated: id \t path)+utt001 /data/audio/001.wav+utt002 /data/audio/002.wav++# train_text.txt (tab-separated: id \t text)+utt001 这是转写文本+utt002 hello world</pre>++<pre># Convert to jsonl+scp2jsonl \+ ++scp_file_list='["/data/list/train_wav.scp", "/data/list/train_text.txt"]' \+ ++data_type_list='["source", "target"]' \+ ++jsonl_file_out="/data/list/train.jsonl"</pre>++<h3>ChatML Format (Fun-ASR-Nano)</h3>+<p>Fun-ASR-Nano uses ChatML conversation format:</p>+<pre>{"messages": [+ {"role": "system", "content": "You are a helpful assistant."},+ {"role": "user", "content": "语音转写:<|startofspeech|>!/path/to/audio.wav<|endofspeech|>"},+ {"role": "assistant", "content": "几点了?"}+], "speech_length": 145, "text_length": 3}</pre>++<table>+<tr><th>Field</th><th>Description</th></tr>+<tr><td><code>messages[0]</code></td><td>System prompt (fixed: "You are a helpful assistant.")</td></tr>+<tr><td><code>messages[1]</code></td><td>User: prompt + audio path wrapped in <code><|startofspeech|>!...<|endofspeech|></code></td></tr>+<tr><td><code>messages[2]</code></td><td>Assistant: transcription text</td></tr>+<tr><td><code>speech_length</code></td><td>Number of fbank frames (10ms each)</td></tr>+<tr><td><code>text_length</code></td><td>Number of tokens (tokenized by Qwen3-0.6B)</td></tr>+</table>++<div class="tip"><strong>Prompt variations:</strong><br>+• Chinese: <code>语音转写:</code><br>+• English: <code>Speech transcription:</code><br>+• Cross-language: <code>语音转写成英文:</code><br>+• No ITN: <code>语音转写,不进行文本规整:</code></div>++<p>Convert from wav.scp + text.txt:</p>+<pre>python tools/scp2jsonl.py \+ ++scp_file=data/train_wav.scp \+ ++transcript_file=data/train_text.txt \+ ++jsonl_file=data/train_example.jsonl</pre>++<!-- Paraformer -->+<h2 id="paraformer">Fine-tune Paraformer</h2>+<pre>cd examples/industrial_data_pretraining/paraformer+bash finetune.sh</pre>++<p>Or customize the key parameters:</p>+<pre>export CUDA_VISIBLE_DEVICES="0,1"+gpu_num=2++torchrun --nproc_per_node $gpu_num \+ funasr/bin/train_ds.py \+ ++model="iic/speech_paraformer-large_asr_nat-zh-cn-16k-common-vocab8404-pytorch" \+ ++train_data_set_list="data/train.jsonl" \+ ++valid_data_set_list="data/val.jsonl" \+ ++dataset_conf.batch_size=6000 \+ ++dataset_conf.batch_type="token" \+ ++dataset_conf.num_workers=4 \+ ++train_conf.max_epoch=50 \+ ++train_conf.validate_interval=2000 \+ ++train_conf.save_checkpoint_interval=2000 \+ ++train_conf.keep_nbest_models=20 \+ ++train_conf.avg_nbest_model=10 \+ ++optim_conf.lr=0.0002 \+ ++output_dir="./outputs"</pre>++<!-- SenseVoice -->+<h2 id="sensevoice">Fine-tune SenseVoice</h2>+<pre>cd examples/industrial_data_pretraining/sense_voice+bash finetune.sh</pre>++<p>Same data format as Paraformer (source/target JSONL). Key difference: SenseVoice uses its own dataset class internally.</p>++<!-- Fun-ASR-Nano -->+<h2 id="nano">Fine-tune Fun-ASR-Nano</h2>+<pre>cd examples/industrial_data_pretraining/fun_asr_nano+bash finetune.sh</pre>++<p>Key differences from Paraformer:</p>+<ul>+<li>Uses ChatML data format (see above)</li>+<li><code>++trust_remote_code=true</code> required</li>+<li>Supports <strong>selective freezing</strong>: freeze encoder/adaptor while training LLM decoder</li>+</ul>++<pre># Freeze encoder + adaptor, only train LLM (recommended for domain adaptation)+++audio_encoder_conf.freeze=true+++audio_adaptor_conf.freeze=true+++llm_conf.freeze=false++# Full fine-tune (all parameters)+++audio_encoder_conf.freeze=false+++audio_adaptor_conf.freeze=false+++llm_conf.freeze=false</pre>++<div class="note"><strong>Recommended strategy:</strong> Start with LLM-only fine-tuning (faster, less data needed). If results are insufficient, unfreeze adaptor. Only unfreeze encoder with very large datasets (>1000h).</div>++<!-- Parameters -->+<h2 id="params">Parameter Reference</h2>++<h3>Dataset Parameters</h3>+<table>+<tr><th>Parameter</th><th>Default</th><th>Description</th></tr>+<tr><td><code>dataset_conf.batch_type</code></td><td>"token"</td><td><code>"token"</code>: dynamic batch by total tokens. <code>"example"</code>: fixed batch count.</td></tr>+<tr><td><code>dataset_conf.batch_size</code></td><td>6000</td><td>Token mode: total frames per batch. Example mode: number of samples.</td></tr>+<tr><td><code>dataset_conf.sort_size</code></td><td>1024</td><td>Buffer size for length-based sorting (improves padding efficiency).</td></tr>+<tr><td><code>dataset_conf.num_workers</code></td><td>4</td><td>Data loading threads.</td></tr>+<tr><td><code>dataset_conf.data_split_num</code></td><td>1</td><td>Split data into N groups for large-scale training (reduces memory).</td></tr>+<tr><td><code>dataset_conf.max_token_length</code></td><td>—</td><td>Filter: skip samples longer than this (in frames/tokens).</td></tr>+<tr><td><code>dataset_conf.min_token_length</code></td><td>—</td><td>Filter: skip samples shorter than this.</td></tr>+</table>++<h3>Training Parameters</h3>+<table>+<tr><th>Parameter</th><th>Default</th><th>Description</th></tr>+<tr><td><code>train_conf.max_epoch</code></td><td>50</td><td>Total training epochs.</td></tr>+<tr><td><code>train_conf.log_interval</code></td><td>1</td><td>Print loss every N steps.</td></tr>+<tr><td><code>train_conf.validate_interval</code></td><td>2000</td><td>Run validation every N steps.</td></tr>+<tr><td><code>train_conf.save_checkpoint_interval</code></td><td>2000</td><td>Save model every N steps.</td></tr>+<tr><td><code>train_conf.keep_nbest_models</code></td><td>20</td><td>Keep top N models (by validation accuracy).</td></tr>+<tr><td><code>train_conf.avg_nbest_model</code></td><td>10</td><td>Average top N models for final checkpoint.</td></tr>+<tr><td><code>train_conf.resume</code></td><td>true</td><td>Resume from last checkpoint if exists.</td></tr>+<tr><td><code>train_conf.use_deepspeed</code></td><td>false</td><td>Enable DeepSpeed ZeRO optimization.</td></tr>+<tr><td><code>optim_conf.lr</code></td><td>0.0002</td><td>Learning rate.</td></tr>+</table>++<!-- Multi-GPU -->+<h2 id="multi-gpu">Multi-GPU Training</h2>++<h3>Single Machine, Multiple GPUs</h3>+<pre>export CUDA_VISIBLE_DEVICES="0,1,2,3"+gpu_num=4++torchrun --nnodes 1 --nproc_per_node $gpu_num \+ funasr/bin/train_ds.py ${train_args}</pre>++<h3>Multiple Machines</h3>+<pre># Machine 1 (master, IP=192.168.1.1)+torchrun --nnodes 2 --node_rank 0 --nproc_per_node 4 \+ --master_addr=192.168.1.1 --master_port=12345 \+ funasr/bin/train_ds.py ${train_args}++# Machine 2+torchrun --nnodes 2 --node_rank 1 --nproc_per_node 4 \+ --master_addr=192.168.1.1 --master_port=12345 \+ funasr/bin/train_ds.py ${train_args}</pre>++<!-- DeepSpeed -->+<h2 id="deepspeed">DeepSpeed</h2>+<p>For large models (Fun-ASR-Nano 800M params), enable DeepSpeed ZeRO:</p>+<pre>++train_conf.use_deepspeed=true+++train_conf.deepspeed_config=./deepspeed_conf/ds_stage1.json</pre>++<p>Stage 1 config (recommended starting point):</p>+<pre>{+ "train_micro_batch_size_per_gpu": 1,+ "gradient_accumulation_steps": 1,+ "bf16": {"enabled": true},+ "zero_optimization": {+ "stage": 1,+ "reduce_bucket_size": 5e8,+ "allgather_bucket_size": 5e8+ }+}</pre>++<div class="note"><strong>When to use which stage:</strong><br>+• Stage 1: Optimizer state partitioned. Good for most cases.<br>+• Stage 2: + Gradient partitioned. For larger models.<br>+• Stage 3: + Parameter partitioned. Maximum memory savings but slower.</div>++<!-- Monitor -->+<h2 id="monitor">Monitoring Training</h2>++<h4>Log file</h4>+<pre>tail -f outputs/log.txt++# Example output:+# train, rank: 0, epoch: 0/50, step: 6990, (loss_avg_rank: 0.327),+# (acc_avg_epoch: 0.795), (lr: 1.165e-04),+# GPU memory: usage: 3.8GB, peak: 18.3GB</pre>++<p>Key metrics to watch:</p>+<ul>+<li><code>loss_avg_epoch</code>: should decrease over time</li>+<li><code>acc_avg_epoch</code>: should increase (most important metric)</li>+<li><code>lr</code>: learning rate at current step</li>+<li><code>GPU memory</code>: peak should not exceed your GPU VRAM</li>+</ul>++<h4>TensorBoard</h4>+<pre>tensorboard --logdir outputs/log/tensorboard+# Open http://localhost:6006</pre>++<!-- Use Fine-tuned Model -->+<h2 id="inference-after">Use Your Fine-tuned Model</h2>++<h4>If outputs/ has configuration.json</h4>+<pre>from funasr import AutoModel+model = AutoModel(model="./outputs")+res = model.generate(input="test.wav")+print(res[0]["text"])</pre>++<h4>If no configuration.json</h4>+<pre>funasr ++model="./outputs" \+ ++config-path="./outputs" \+ ++config-name="config.yaml" \+ ++init_param="./outputs/model.pt" \+ ++input="test.wav"</pre>++<!-- Tips -->+<h2 id="tips">Tips & Troubleshooting</h2>++<h4>OOM during training</h4>+<ol>+<li>Reduce <code>dataset_conf.batch_size</code></li>+<li>Add <code>dataset_conf.max_token_length=2000</code> to filter long utterances</li>+<li>Enable DeepSpeed (partitions optimizer states)</li>+<li>Reduce <code>dataset_conf.num_workers</code></li>+</ol>++<h4>Training loss stuck / NaN gradients</h4>+<ul>+<li>Reduce learning rate (try 0.00005)</li>+<li>Check data quality — corrupted audio files cause NaN</li>+<li>For Fun-ASR-Nano: start with encoder frozen</li>+</ul>++<h4>Validation accuracy not improving</h4>+<ul>+<li>Increase training data (min ~10h for fine-tuning)</li>+<li>Check domain match — model may not generalize to very different domains</li>+<li>Try unfreezing more layers gradually</li>+</ul>++<h4>Large-scale data (>10,000 hours)</h4>+<p>Use data splitting to avoid memory issues:</p>+<pre># Split data into chunks, load 2 at a time+++dataset_conf.data_split_num=256+# data.list contains paths to split jsonl files:+# data/train.0.jsonl+# data/train.1.jsonl+# ...+++train_data_set_list="data/data.list"</pre>++<h4>Resume after crash</h4>+<p>Set <code>++train_conf.resume=true</code> (default). Training automatically restarts from the latest checkpoint in <code>output_dir</code>.</p>++</div></div>++<footer>+<p>FunASR · Tongyi Lab, Alibaba Group</p>+<p><a href="index.html">Home</a> · <a href="tutorial.html">Tutorial</a> · <a href="model-registration.html">Develop</a> · <a href="api.html">API</a> · <a href="https://github.com/modelscope/FunASR">GitHub</a></p>+</footer>+</body></html>diff --git a/scripts/gen_api_docs.py b/scripts/gen_api_docs.pyindex 09cf5e3c1..7b4bc3823 100644--- a/scripts/gen_api_docs.py+++ b/scripts/gen_api_docs.py@@ -1,5 +1,4 @@#!/usr/bin/env python3-import shutil"""Generate API page with source code preview and GitHub links.Run this script from the repo root on main branch.Output: gh-pages-output/api.html for deployment to gh-pages.@@ -7,12 +6,20 @@ Output: gh-pages-output/api.html for deployment to gh-pages.This script is designed to be run by GitHub Actions on every push to main,automatically regenerating the API docs."""-import ast, os, html as htmlmod, re, glob, inspect+import ast+import glob+import html as htmlmod+import os+import re+import shutil+from pathlib import PathREPO_URL = "https://github.com/modelscope/FunASR"BRANCH = "main"+REPO_ROOT = Path(__file__).resolve().parents[1]+OUTPUT_DIR = REPO_ROOT / "gh-pages-output"-os.chdir('/cpfs_speech/user/zhifu.gzf/codebase/FunASR-github')+os.chdir(REPO_ROOT)tree_structure = {"funasr.auto": {"auto_model": "funasr/auto/auto_model.py"},@@ -113,7 +120,7 @@ for top_level, sub_modules in tree_structure.items():with open(filepath) as f:source = f.read()try:- tree = ast.parse(source)+ tree = ast.parse(source, filename=filepath)except:continuekey = f"{top_level}.{sub_name}"@@ -213,10 +220,7 @@ for top_level in tree_structure:sidebar += '</div></div>\n'sidebar += '</div></div>\n'-# Write the full page (keeping same CSS structure + new source styles)-page = open('/tmp/api_template.txt', 'w') # We'll inline it-page.close()-+# Write the full page (keeping same CSS structure + source styles)html_page = f'''<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8"><meta name="viewport" content="width=device-width,initial-scale=1.0"><title>FunASR API</title>@@ -267,7 +271,7 @@ h2{{font-size:1.1rem;margin:8px 0}}h2 code{{font-family:'JetBrains Mono',monospa</style></head><body><nav class="nav"><div class="container"><a href="index.html" class="nav-logo">FunASR</a>-<div class="nav-links"><a href="index.html">Home</a><a href="tutorial.html">Tutorial</a><a href="model-registration.html">Develop</a><a href="api.html" class="active">API</a></div>+<div class="nav-links"><a href="index.html">Home</a><a href="tutorial.html">Tutorial</a><a href="training.html">Training</a><a href="model-registration.html">Develop</a><a href="api.html" class="active">API</a></div><a href="https://github.com/modelscope/FunASR" class="nav-github">GitHub</a></div></nav><div class="api-layout">@@ -298,10 +302,12 @@ if(q)document.querySelectorAll('.l1,.l2').forEach(e=>e.classList.add('open'));if(window.location.hash)showEntry(window.location.hash.slice(1));</script></body></html>'''-with open("gh-pages-output/api.html", "w") as f:+OUTPUT_DIR.mkdir(parents=True, exist_ok=True)+with (OUTPUT_DIR / "api.html").open("w", encoding="utf-8") as f:f.write(html_page)-print("Generated gh-pages-output/api.html")-# Ensure output dir exists-os.makedirs("gh-pages-output", exist_ok=True)+training_src = REPO_ROOT / "training.html"+if training_src.exists():+ shutil.copy2(training_src, OUTPUT_DIR / "training.html")+print("Generated gh-pages-output/api.html")