Open-source speech recognition toolkit for training, inference, streaming ASR, VAD, punctuation, speaker diarization pipelines, and OpenAI-compatible/MCP serving.
Python asraudiochineseemotion-recognitionfunasrmcp-servermultilingual-asropenai-compatible-apiparaformerpunctuationpytorchreal-time-asrspeaker-diarizationspeech-recognitionspeech-to-textstreaming-asrtranscriptionvllmvoice-activity-detectionwhisper-alternative
add whisper_lib for sensevoice (#2735)
彭震东1bf72e7ff8d3
diff --git a/funasr/models/sense_voice/whisper_lib/__init__.py b/funasr/models/sense_voice/whisper_lib/__init__.pynew file mode 100644index 000000000..855b8cf26--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/__init__.py@@ -0,0 +1,152 @@+import hashlib+import io+import os+import urllib+import warnings+from typing import List, Optional, Union++import torch+from tqdm import tqdm++from .audio import load_audio, log_mel_spectrogram, pad_or_trim+from .decoding import DecodingOptions, DecodingResult, decode, detect_language+from .model import ModelDimensions, Whisper+from .transcribe import transcribe+from .version import __version__++_MODELS = {+ "tiny.en": "https://openaipublic.azureedge.net/main/whisper/models/d3dd57d32accea0b295c96e26691aa14d8822fac7d9d27d5dc00b4ca2826dd03/tiny.en.pt",+ "tiny": "https://openaipublic.azureedge.net/main/whisper/models/65147644a518d12f04e32d6f3b26facc3f8dd46e5390956a9424a650c0ce22b9/tiny.pt",+ "base.en": "https://openaipublic.azureedge.net/main/whisper/models/25a8566e1d0c1e2231d1c762132cd20e0f96a85d16145c3a00adf5d1ac670ead/base.en.pt",+ "base": "https://openaipublic.azureedge.net/main/whisper/models/ed3a0b6b1c0edf879ad9b11b1af5a0e6ab5db9205f891f668f8b0e6c6326e34e/base.pt",+ "small.en": "https://openaipublic.azureedge.net/main/whisper/models/f953ad0fd29cacd07d5a9eda5624af0f6bcf2258be67c92b79389873d91e0872/small.en.pt",+ "small": "https://openaipublic.azureedge.net/main/whisper/models/9ecf779972d90ba49c06d968637d720dd632c55bbf19d441fb42bf17a411e794/small.pt",+ "medium.en": "https://openaipublic.azureedge.net/main/whisper/models/d7440d1dc186f76616474e0ff0b3b6b879abc9d1a4926b7adfa41db2d497ab4f/medium.en.pt",+ "medium": "https://openaipublic.azureedge.net/main/whisper/models/345ae4da62f9b3d59415adc60127b97c714f32e89e936602e85993674d08dcb1/medium.pt",+ "large-v1": "https://openaipublic.azureedge.net/main/whisper/models/e4b87e7e0bf463eb8e6956e646f1e277e901512310def2c24bf0e11bd3c28e9a/large-v1.pt",+ "large-v2": "https://openaipublic.azureedge.net/main/whisper/models/81f7c96c852ee8fc832187b0132e569d6c3065a3252ed18e56effd0b6a73e524/large-v2.pt",+ "large-v3": "https://openaipublic.azureedge.net/main/whisper/models/e5b1a55b89c1367dacf97e3e19bfd829a01529dbfdeefa8caeb59b3f1b81dadb/large-v3.pt",+ "large": "https://openaipublic.azureedge.net/main/whisper/models/e5b1a55b89c1367dacf97e3e19bfd829a01529dbfdeefa8caeb59b3f1b81dadb/large-v3.pt",+}++# base85-encoded (n_layers, n_heads) boolean arrays indicating the cross-attention heads that are+# highly correlated to the word-level timing, i.e. the alignment between audio and text tokens.+_ALIGNMENT_HEADS = {+ "tiny.en": b"ABzY8J1N>@0{>%R00Bk>$p{7v037`oCl~+#00",+ "tiny": b"ABzY8bu8Lr0{>%RKn9Fp%m@SkK7Kt=7ytkO",+ "base.en": b"ABzY8;40c<0{>%RzzG;p*o+Vo09|#PsxSZm00",+ "base": b"ABzY8KQ!870{>%RzyTQH3`Q^yNP!>##QT-<FaQ7m",+ "small.en": b"ABzY8>?_)10{>%RpeA61k&I|OI3I$65C{;;pbCHh0B{qLQ;+}v00",+ "small": b"ABzY8DmU6=0{>%Rpa?J`kvJ6qF(V^F86#Xh7JUGMK}P<N0000",+ "medium.en": b"ABzY8usPae0{>%R7<zz_OvQ{)4kMa0BMw6u5rT}kRKX;$NfYBv00*Hl@qhsU00",+ "medium": b"ABzY8B0Jh+0{>%R7}kK1fFL7w6%<-Pf*t^=N)Qr&0RR9",+ "large-v1": b"ABzY8r9j$a0{>%R7#4sLmoOs{s)o3~84-RPdcFk!JR<kSfC2yj",+ "large-v2": b"ABzY8zd+h!0{>%R7=D0pU<_bnWW*tkYAhobTNnu$jnkEkXqp)j;w1Tzk)UH3X%SZd&fFZ2fC2yj",+ "large-v3": b"ABzY8gWO1E0{>%R7(9S+Kn!D~%ngiGaR?*L!iJG9p-nab0JQ=-{D1-g00",+ "large": b"ABzY8gWO1E0{>%R7(9S+Kn!D~%ngiGaR?*L!iJG9p-nab0JQ=-{D1-g00",+}+++def _download(url: str, root: str, in_memory: bool) -> Union[bytes, str]:+ os.makedirs(root, exist_ok=True)++ expected_sha256 = url.split("/")[-2]+ download_target = os.path.join(root, os.path.basename(url))++ if os.path.exists(download_target) and not os.path.isfile(download_target):+ raise RuntimeError(f"{download_target} exists and is not a regular file")++ if os.path.isfile(download_target):+ with open(download_target, "rb") as f:+ model_bytes = f.read()+ if hashlib.sha256(model_bytes).hexdigest() == expected_sha256:+ return model_bytes if in_memory else download_target+ else:+ warnings.warn(+ f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file"+ )++ with urllib.request.urlopen(url) as source, open(download_target, "wb") as output:+ with tqdm(+ total=int(source.info().get("Content-Length")),+ ncols=80,+ unit="iB",+ unit_scale=True,+ unit_divisor=1024,+ ) as loop:+ while True:+ buffer = source.read(8192)+ if not buffer:+ break++ output.write(buffer)+ loop.update(len(buffer))++ model_bytes = open(download_target, "rb").read()+ if hashlib.sha256(model_bytes).hexdigest() != expected_sha256:+ raise RuntimeError(+ "Model has been downloaded but the SHA256 checksum does not not match. Please retry loading the model."+ )++ return model_bytes if in_memory else download_target+++def available_models() -> List[str]:+ """Returns the names of available models"""+ return list(_MODELS.keys())+++def load_model(+ name: str,+ device: Optional[Union[str, torch.device]] = None,+ download_root: str = None,+ in_memory: bool = False,+) -> Whisper:+ """+ Load a Whisper ASR model++ Parameters+ ----------+ name : str+ one of the official model names listed by `whisper.available_models()`, or+ path to a model checkpoint containing the model dimensions and the model state_dict.+ device : Union[str, torch.device]+ the PyTorch device to put the model into+ download_root: str+ path to download the model files; by default, it uses "~/.cache/whisper"+ in_memory: bool+ whether to preload the model weights into host memory++ Returns+ -------+ model : Whisper+ The Whisper ASR model instance+ """++ if device is None:+ device = "cuda" if torch.cuda.is_available() else "cpu"+ if download_root is None:+ default = os.path.join(os.path.expanduser("~"), ".cache")+ download_root = os.path.join(os.getenv("XDG_CACHE_HOME", default), "whisper")++ if name in _MODELS:+ checkpoint_file = _download(_MODELS[name], download_root, in_memory)+ alignment_heads = _ALIGNMENT_HEADS[name]+ elif os.path.isfile(name):+ checkpoint_file = open(name, "rb").read() if in_memory else name+ alignment_heads = None+ else:+ raise RuntimeError(f"Model {name} not found; available models = {available_models()}")++ with io.BytesIO(checkpoint_file) if in_memory else open(checkpoint_file, "rb") as fp:+ checkpoint = torch.load(fp, map_location=device)+ del checkpoint_file++ dims = ModelDimensions(**checkpoint["dims"])+ model = Whisper(dims)+ model.load_state_dict(checkpoint["model_state_dict"])++ if alignment_heads is not None:+ model.set_alignment_heads(alignment_heads)++ return model.to(device)diff --git a/funasr/models/sense_voice/whisper_lib/__main__.py b/funasr/models/sense_voice/whisper_lib/__main__.pynew file mode 100644index 000000000..8874c7aff--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/__main__.py@@ -0,0 +1,3 @@+# from .transcribe import cli+#+# cli()diff --git a/funasr/models/sense_voice/whisper_lib/audio.py b/funasr/models/sense_voice/whisper_lib/audio.pynew file mode 100644index 000000000..2f688e90b--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/audio.py@@ -0,0 +1,155 @@+import os+from functools import lru_cache+from subprocess import CalledProcessError, run+from typing import Optional, Union++import numpy as np+import torch+import torch.nn.functional as F++from .utils import exact_div++# hard-coded audio hyperparameters+SAMPLE_RATE = 16000+N_FFT = 400+HOP_LENGTH = 160+CHUNK_LENGTH = 30+N_SAMPLES = CHUNK_LENGTH * SAMPLE_RATE # 480000 samples in a 30-second chunk+N_FRAMES = exact_div(N_SAMPLES, HOP_LENGTH) # 3000 frames in a mel spectrogram input++N_SAMPLES_PER_TOKEN = HOP_LENGTH * 2 # the initial convolutions has stride 2+FRAMES_PER_SECOND = exact_div(SAMPLE_RATE, HOP_LENGTH) # 10ms per audio frame+TOKENS_PER_SECOND = exact_div(SAMPLE_RATE, N_SAMPLES_PER_TOKEN) # 20ms per audio token+++def load_audio(file: str, sr: int = SAMPLE_RATE):+ """+ Open an audio file and read as mono waveform, resampling as necessary++ Parameters+ ----------+ file: str+ The audio file to open++ sr: int+ The sample rate to resample the audio if necessary++ Returns+ -------+ A NumPy array containing the audio waveform, in float32 dtype.+ """++ # This launches a subprocess to decode audio while down-mixing+ # and resampling as necessary. Requires the ffmpeg CLI in PATH.+ # fmt: off+ cmd = [+ "ffmpeg",+ "-nostdin",+ "-threads", "0",+ "-i", file,+ "-f", "s16le",+ "-ac", "1",+ "-acodec", "pcm_s16le",+ "-ar", str(sr),+ "-"+ ]+ # fmt: on+ try:+ out = run(cmd, capture_output=True, check=True).stdout+ except CalledProcessError as e:+ raise RuntimeError(f"Failed to load audio: {e.stderr.decode()}") from e++ return np.frombuffer(out, np.int16).flatten().astype(np.float32) / 32768.0+++def pad_or_trim(array, length: int = N_SAMPLES, *, axis: int = -1):+ """+ Pad or trim the audio array to N_SAMPLES, as expected by the encoder.+ """+ if torch.is_tensor(array):+ if array.shape[axis] > length:+ array = array.index_select(dim=axis, index=torch.arange(length, device=array.device))++ if array.shape[axis] < length:+ pad_widths = [(0, 0)] * array.ndim+ pad_widths[axis] = (0, length - array.shape[axis])+ array = F.pad(array, [pad for sizes in pad_widths[::-1] for pad in sizes])+ else:+ if array.shape[axis] > length:+ array = array.take(indices=range(length), axis=axis)++ if array.shape[axis] < length:+ pad_widths = [(0, 0)] * array.ndim+ pad_widths[axis] = (0, length - array.shape[axis])+ array = np.pad(array, pad_widths)++ return array+++@lru_cache(maxsize=None)+def mel_filters(device, n_mels: int, filters_path: str = None) -> torch.Tensor:+ """+ load the mel filterbank matrix for projecting STFT into a Mel spectrogram.+ Allows decoupling librosa dependency; saved using:++ np.savez_compressed(+ "mel_filters.npz",+ mel_80=librosa.filters.mel(sr=16000, n_fft=400, n_mels=80),+ mel_128=librosa.filters.mel(sr=16000, n_fft=400, n_mels=128),+ )+ """+ assert n_mels in {80, 128}, f"Unsupported n_mels: {n_mels}"+ if filters_path is None:+ filters_path = os.path.join(os.path.dirname(__file__), "assets", "mel_filters.npz")+ with np.load(filters_path, allow_pickle=False) as f:+ return torch.from_numpy(f[f"mel_{n_mels}"]).to(device)+++def log_mel_spectrogram(+ audio: Union[str, np.ndarray, torch.Tensor],+ n_mels: int = 80,+ padding: int = 0,+ device: Optional[Union[str, torch.device]] = None,+):+ """+ Compute the log-Mel spectrogram of++ Parameters+ ----------+ audio: Union[str, np.ndarray, torch.Tensor], shape = (*)+ The path to audio or either a NumPy array or Tensor containing the audio waveform in 16 kHz++ n_mels: int+ The number of Mel-frequency filters, only 80 is supported++ padding: int+ Number of zero samples to pad to the right++ device: Optional[Union[str, torch.device]]+ If given, the audio tensor is moved to this device before STFT++ Returns+ -------+ torch.Tensor, shape = (80, n_frames)+ A Tensor that contains the Mel spectrogram+ """+ if not torch.is_tensor(audio):+ if isinstance(audio, str):+ audio = load_audio(audio)+ audio = torch.from_numpy(audio)++ if device is not None:+ audio = audio.to(device)+ if padding > 0:+ audio = F.pad(audio, (0, padding))+ window = torch.hann_window(N_FFT).to(audio.device)+ stft = torch.stft(audio, N_FFT, HOP_LENGTH, window=window, return_complex=True)+ magnitudes = stft[..., :-1].abs() ** 2++ filters = mel_filters(audio.device, n_mels)+ mel_spec = filters @ magnitudes++ log_spec = torch.clamp(mel_spec, min=1e-10).log10()+ log_spec = torch.maximum(log_spec, log_spec.max() - 8.0)+ log_spec = (log_spec + 4.0) / 4.0+ return log_specdiff --git a/funasr/models/sense_voice/whisper_lib/decoding.py b/funasr/models/sense_voice/whisper_lib/decoding.pynew file mode 100644index 000000000..644755f5f--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/decoding.py@@ -0,0 +1,892 @@+from dataclasses import dataclass, field, replace+from typing import TYPE_CHECKING, Dict, Iterable, List, Optional, Sequence, Tuple, Union++import numpy as np+import torch+import torch.nn.functional as F+from torch import Tensor+from torch.distributions import Categorical++from .audio import CHUNK_LENGTH+from .tokenizer import Tokenizer, get_tokenizer+from .utils import compression_ratio+from funasr.models.transformer.utils.nets_utils import to_device+++if TYPE_CHECKING:+ from .model import Whisper++[email protected]_grad()+def detect_language(+ model: "Whisper",+ mel: Tensor,+ tokenizer: Tokenizer = None,+ initial_prompt=None,+ x=None,+) -> Tuple[Tensor, List[dict]]:+ """+ Detect the spoken language in the audio, and return them as list of strings, along with the ids+ of the most probable language tokens and the probability distribution over all language tokens.+ This is performed outside the main decode loop in order to not interfere with kv-caching.++ Returns+ -------+ language_tokens : Tensor, shape = (n_audio,)+ ids of the most probable language tokens, which appears after the startoftranscript token.+ language_probs : List[Dict[str, float]], length = n_audio+ list of dictionaries containing the probability distribution over all languages.+ """+ if tokenizer is None:+ tokenizer = get_tokenizer(model.is_multilingual, num_languages=model.num_languages)+ if tokenizer.language is None or tokenizer.language_token not in tokenizer.sot_sequence:+ raise ValueError("This model doesn't have language tokens so it can't perform lang id")++ single = mel.ndim == 2+ if single:+ mel = mel.unsqueeze(0)++ # skip encoder forward pass if already-encoded audio features were given+ # FIX(funasr): sense vocie+ if mel.shape[-1] != model.dims.n_audio_state:+ mel = model.encoder(mel)++ # forward pass using a single token, startoftranscript+ n_audio = mel.shape[0]+ # FIX(funasr): sense vocie+ # x = torch.tensor([[tokenizer.sot]] * n_audio).to(mel.device) # [n_audio, 1]+ if x is None:+ x = torch.tensor([tokenizer.encode(initial_prompt, allowed_special="all")] * n_audio).to(+ mel.device+ ) # [n_audio, 1]++ else:+ x = x.to(mel.device)+ # FIX(funasr): sense vocie+ logits = model.logits(x[:, :-1], mel)[:, -1]+ # logits = model.logits(x[:, :], mel)[:, -1]++ # collect detected languages; suppress all non-language tokens+ mask = torch.ones(logits.shape[-1], dtype=torch.bool)+ mask[list(tokenizer.all_language_tokens)] = False+ mask[tokenizer.no_speech] = False++ logits[:, mask] = -np.inf+ language_tokens = logits.argmax(dim=-1)+ language_token_probs = logits.softmax(dim=-1).cpu()++ language_probs = [+ {+ c: language_token_probs[i, j].item()+ for j, c in zip(+ list(tokenizer.all_language_tokens) + [tokenizer.no_speech],+ list(tokenizer.all_language_codes) + ["nospeech"],+ )+ }+ for i in range(n_audio)+ ]++ if single:+ language_tokens = language_tokens[0]+ language_probs = language_probs[0]++ return language_tokens, language_probs+++@dataclass(frozen=True)+class DecodingOptions:+ # whether to perform X->X "transcribe" or X->English "translate"+ task: str = "transcribe"++ # language that the audio is in; uses detected language if None+ language: Optional[str] = None++ # sampling-related options+ temperature: float = 0.0+ sample_len: Optional[int] = None # maximum number of tokens to sample+ best_of: Optional[int] = None # number of independent sample trajectories, if t > 0+ beam_size: Optional[int] = None # number of beams in beam search, if t == 0+ patience: Optional[float] = None # patience in beam search (arxiv:2204.05424)++ # "alpha" in Google NMT, or None for length norm, when ranking generations+ # to select which to return among the beams or best-of-N samples+ length_penalty: Optional[float] = None++ # text or tokens to feed as the prompt or the prefix; for more info:+ # https://github.com/openai/whisper/discussions/117#discussioncomment-3727051+ prompt: Optional[Union[str, List[int]]] = None # for the previous context+ prefix: Optional[Union[str, List[int]]] = None # to prefix the current context++ # list of tokens ids (or comma-separated token ids) to suppress+ # "-1" will suppress a set of symbols as defined in `tokenizer.non_speech_tokens()`+ suppress_tokens: Optional[Union[str, Iterable[int]]] = "-1"+ suppress_blank: bool = True # this will suppress blank outputs++ gain_event: bool = False # this will suppress blank outputs+ gain_tokens_bg: Optional[Union[str, List[int]]] = "<|Speech|><|BGM|><|Applause|><|Laughter|>"+ gain_tokens_ed: Optional[Union[str, List[int]]] = (+ "<|/Speech|><|/BGM|><|/Applause|><|/Laughter|>"+ )+ gain_tokens_score: List[float] = field(default_factory=lambda: [1, 1, 25.0, 5.0]) # [25, 5]++ use_emo_threshold: bool = False # this will suppress blank outputs+ emo_unk_token: Optional[Union[str, List[int]]] = "<|SPECIAL_TOKEN_1|>"+ emo_target_tokens: Optional[Union[str, List[int]]] = "<|HAPPY|><|SAD|><|ANGRY|>"+ emo_target_threshold: List[float] = field(default_factory=lambda: [0.1, 0.1, 0.1]) # [25, 5]++ # timestamp sampling options+ without_timestamps: bool = False # use <|notimestamps|> to sample text tokens only+ max_initial_timestamp: Optional[float] = 1.0++ # implementation details+ fp16: bool = True # use fp16 for most of the calculation++ # FIX(funasr): sense vocie+ initial_prompt: str = None+ vocab_path: str = None+++@dataclass(frozen=True)+class DecodingResult:+ audio_features: Tensor+ language: str+ language_probs: Optional[Dict[str, float]] = None+ tokens: List[int] = field(default_factory=list)+ text: str = ""+ avg_logprob: float = np.nan+ no_speech_prob: float = np.nan+ temperature: float = np.nan+ compression_ratio: float = np.nan+++class Inference:+ def logits(self, tokens: Tensor, audio_features: Tensor) -> Tensor:+ """Perform a forward pass on the decoder and return per-token logits"""+ raise NotImplementedError++ def rearrange_kv_cache(self, source_indices) -> None:+ """Update the key-value cache according to the updated beams"""+ raise NotImplementedError++ def cleanup_caching(self) -> None:+ """Clean up any resources or hooks after decoding is finished"""+ pass+++class PyTorchInference(Inference):+ def __init__(self, model: "Whisper", initial_token_length: int):+ self.model: "Whisper" = model+ self.initial_token_length = initial_token_length+ self.kv_cache = {}+ self.hooks = []++ key_modules = [block.attn.key for block in self.model.decoder.blocks]+ value_modules = [block.attn.value for block in self.model.decoder.blocks]+ self.kv_modules = key_modules + value_modules++ def logits(self, tokens: Tensor, audio_features: Tensor) -> Tensor:+ if not self.kv_cache:+ self.kv_cache, self.hooks = self.model.install_kv_cache_hooks()++ if tokens.shape[-1] > self.initial_token_length:+ # only need to use the last token except in the first forward pass+ tokens = tokens[:, -1:]++ return self.model.decoder(tokens, audio_features, kv_cache=self.kv_cache)++ def cleanup_caching(self):+ for hook in self.hooks:+ hook.remove()++ self.kv_cache = {}+ self.hooks = []++ def rearrange_kv_cache(self, source_indices):+ if source_indices != list(range(len(source_indices))):+ for module in self.kv_modules:+ # update the key/value cache to contain the selected sequences+ self.kv_cache[module] = self.kv_cache[module][source_indices].detach()+++class SequenceRanker:+ def rank(self, tokens: List[List[Tensor]], sum_logprobs: List[List[float]]) -> List[int]:+ """+ Given a list of groups of samples and their cumulative log probabilities,+ return the indices of the samples in each group to select as the final result+ """+ raise NotImplementedError+++class MaximumLikelihoodRanker(SequenceRanker):+ """+ Select the sample with the highest log probabilities, penalized using either+ a simple length normalization or Google NMT paper's length penalty+ """++ def __init__(self, length_penalty: Optional[float]):+ self.length_penalty = length_penalty++ def rank(self, tokens: List[List[Tensor]], sum_logprobs: List[List[float]]):+ def scores(logprobs, lengths):+ result = []+ for logprob, length in zip(logprobs, lengths):+ if self.length_penalty is None:+ penalty = length+ else:+ # from the Google NMT paper+ penalty = ((5 + length) / 6) ** self.length_penalty+ result.append(logprob / penalty)+ return result++ # get the sequence with the highest score+ lengths = [[len(t) for t in s] for s in tokens]+ return [np.argmax(scores(p, l)) for p, l in zip(sum_logprobs, lengths)]+++class TokenDecoder:+ def reset(self):+ """Initialize any stateful variables for decoding a new sequence"""++ def update(self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor) -> Tuple[Tensor, bool]:+ """Specify how to select the next token, based on the current trace and logits++ Parameters+ ----------+ tokens : Tensor, shape = (n_batch, current_sequence_length)+ all tokens in the context so far, including the prefix and sot_sequence tokens++ logits : Tensor, shape = (n_batch, vocab_size)+ per-token logits of the probability distribution at the current step++ sum_logprobs : Tensor, shape = (n_batch)+ cumulative log probabilities for each sequence++ Returns+ -------+ tokens : Tensor, shape = (n_batch, current_sequence_length + 1)+ the tokens, appended with the selected next token++ completed : bool+ True if all sequences has reached the end of text++ """+ raise NotImplementedError++ def finalize(+ self, tokens: Tensor, sum_logprobs: Tensor+ ) -> Tuple[Sequence[Sequence[Tensor]], List[List[float]]]:+ """Finalize search and return the final candidate sequences++ Parameters+ ----------+ tokens : Tensor, shape = (n_audio, n_group, current_sequence_length)+ all tokens in the context so far, including the prefix and sot_sequence++ sum_logprobs : Tensor, shape = (n_audio, n_group)+ cumulative log probabilities for each sequence++ Returns+ -------+ tokens : Sequence[Sequence[Tensor]], length = n_audio+ sequence of Tensors containing candidate token sequences, for each audio input++ sum_logprobs : List[List[float]], length = n_audio+ sequence of cumulative log probabilities corresponding to the above++ """+ raise NotImplementedError+++class GreedyDecoder(TokenDecoder):+ def __init__(self, temperature: float, eot: int):+ self.temperature = temperature+ self.eot = eot++ def update(self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor) -> Tuple[Tensor, bool]:+ if self.temperature == 0:+ next_tokens = logits.argmax(dim=-1)+ else:+ next_tokens = Categorical(logits=logits / self.temperature).sample()++ logprobs = F.log_softmax(logits.float(), dim=-1)+ current_logprobs = logprobs[torch.arange(logprobs.shape[0]), next_tokens]+ sum_logprobs += current_logprobs * (tokens[:, -1] != self.eot)++ next_tokens[tokens[:, -1] == self.eot] = self.eot+ tokens = torch.cat([tokens, next_tokens[:, None]], dim=-1)++ completed = (tokens[:, -1] == self.eot).all()+ return tokens, completed++ def finalize(self, tokens: Tensor, sum_logprobs: Tensor):+ # make sure each sequence has at least one EOT token at the end+ tokens = F.pad(tokens, (0, 1), value=self.eot)+ return tokens, sum_logprobs.tolist()+++class BeamSearchDecoder(TokenDecoder):+ def __init__(+ self,+ beam_size: int,+ eot: int,+ inference: Inference,+ patience: Optional[float] = None,+ ):+ self.beam_size = beam_size+ self.eot = eot+ self.inference = inference+ self.patience = patience or 1.0+ self.max_candidates: int = round(beam_size * self.patience)+ self.finished_sequences = None++ assert self.max_candidates > 0, f"Invalid beam size ({beam_size}) or patience ({patience})"++ def reset(self):+ self.finished_sequences = None++ def update(self, tokens: Tensor, logits: Tensor, sum_logprobs: Tensor) -> Tuple[Tensor, bool]:+ if tokens.shape[0] % self.beam_size != 0:+ raise ValueError(f"{tokens.shape}[0] % {self.beam_size} != 0")++ n_audio = tokens.shape[0] // self.beam_size+ if self.finished_sequences is None: # for the first update+ self.finished_sequences = [{} for _ in range(n_audio)]++ logprobs = F.log_softmax(logits.float(), dim=-1)+ next_tokens, source_indices, finished_sequences = [], [], []+ for i in range(n_audio):+ scores, sources, finished = {}, {}, {}++ # STEP 1: calculate the cumulative log probabilities for possible candidates+ for j in range(self.beam_size):+ idx = i * self.beam_size + j+ prefix = tokens[idx].tolist()+ for logprob, token in zip(*logprobs[idx].topk(self.beam_size + 1)):+ new_logprob = (sum_logprobs[idx] + logprob).item()+ sequence = tuple(prefix + [token.item()])+ scores[sequence] = new_logprob+ sources[sequence] = idx++ # STEP 2: rank the candidates and keep the top beam_size sequences for each audio+ saved = 0+ for sequence in sorted(scores, key=scores.get, reverse=True):+ if sequence[-1] == self.eot:+ finished[sequence] = scores[sequence]+ else:+ sum_logprobs[len(next_tokens)] = scores[sequence]+ next_tokens.append(sequence)+ source_indices.append(sources[sequence])++ saved += 1+ if saved == self.beam_size:+ break++ finished_sequences.append(finished)++ tokens = torch.tensor(next_tokens, device=tokens.device)+ self.inference.rearrange_kv_cache(source_indices)++ # add newly finished sequences to self.finished_sequences+ assert len(self.finished_sequences) == len(finished_sequences)+ for previously_finished, newly_finished in zip(self.finished_sequences, finished_sequences):+ for seq in sorted(newly_finished, key=newly_finished.get, reverse=True):+ if len(previously_finished) >= self.max_candidates:+ break # the candidate list is full+ previously_finished[seq] = newly_finished[seq]++ # mark as completed if all audio has enough number of samples+ completed = all(+ len(sequences) >= self.max_candidates for sequences in self.finished_sequences+ )+ return tokens, completed++ def finalize(self, preceding_tokens: Tensor, sum_logprobs: Tensor):+ # collect all finished sequences, including patience, and add unfinished ones if not enough+ sum_logprobs = sum_logprobs.cpu()+ for i, sequences in enumerate(self.finished_sequences):+ if len(sequences) < self.beam_size: # when not enough sequences are finished+ for j in list(np.argsort(sum_logprobs[i]))[::-1]:+ sequence = preceding_tokens[i, j].tolist() + [self.eot]+ sequences[tuple(sequence)] = sum_logprobs[i][j].item()+ if len(sequences) >= self.beam_size:+ break++ tokens: List[List[Tensor]] = [+ [torch.tensor(seq) for seq in sequences.keys()] for sequences in self.finished_sequences+ ]+ sum_logprobs: List[List[float]] = [+ list(sequences.values()) for sequences in self.finished_sequences+ ]+ return tokens, sum_logprobs+++class LogitFilter:+ def apply(self, logits: Tensor, tokens: Tensor) -> None:+ """Apply any filtering or masking to logits in-place++ Parameters+ ----------+ logits : Tensor, shape = (n_batch, vocab_size)+ per-token logits of the probability distribution at the current step++ tokens : Tensor, shape = (n_batch, current_sequence_length)+ all tokens in the context so far, including the prefix and sot_sequence tokens++ """+ raise NotImplementedError+++class SuppressBlank(LogitFilter):+ def __init__(self, tokenizer: Tokenizer, sample_begin: int):+ self.tokenizer = tokenizer+ self.sample_begin = sample_begin++ def apply(self, logits: Tensor, tokens: Tensor):+ if tokens.shape[1] == self.sample_begin:+ logits[:, self.tokenizer.encode(" ") + [self.tokenizer.eot]] = -np.inf+++class SuppressTokens(LogitFilter):+ def __init__(self, suppress_tokens: Sequence[int]):+ self.suppress_tokens = list(suppress_tokens)++ def apply(self, logits: Tensor, tokens: Tensor):+ logits[:, self.suppress_tokens] = -np.inf+++class GainEventToken(LogitFilter):+ def __init__(+ self, bg_tokens: Sequence[int], ed_tokens: Sequence[int], gain_values: Sequence[float]+ ):+ self.bg_tokens = list(bg_tokens)+ self.ed_tokens = list(ed_tokens)+ self.gain_value = [np.log(max(ga, 1e-9)) for ga in gain_values]+ assert len(self.ed_tokens) == len(self.gain_value)+ assert len(self.bg_tokens) == len(self.gain_value)++ def apply(self, logits: Tensor, tokens: Tensor):+ for i in range(len(tokens)):+ for bg, ed, ga in zip(self.bg_tokens, self.ed_tokens, self.gain_value):+ sum_bg = sum([1 if x == bg else 0 for x in tokens[i]])+ sum_ed = sum([1 if x == ed else 0 for x in tokens[i]])+ logits[i, bg] += ga+ if sum_bg > sum_ed or tokens[i, -1] in [bg, ed]:+ logits[i, bg] = -np.inf+ if sum_bg <= sum_ed:+ logits[i, ed] = -np.inf+++class ThresholdEmoToken(LogitFilter):+ def __init__(+ self, unk_tokens: Sequence[int], emo_tokens: Sequence[int], th_values: Sequence[float]+ ):+ self.unk_token = list(unk_tokens)[0]+ self.emo_tokens = list(emo_tokens)+ self.th_values = list(th_values)+ assert len(self.emo_tokens) == len(self.th_values)++ def apply(self, logits: Tensor, tokens: Tensor):+ for i in range(len(tokens)):+ for emo, th in zip(self.emo_tokens, self.th_values):+ if logits[i].argmax() == emo and logits[i].softmax(dim=-1)[emo] < th:+ logits[i, self.unk_token] = max(logits[i, emo], logits[i, self.unk_token])+ logits[i, emo] = -np.inf++ # for bg, ed, ga in zip(self.bg_tokens, self.ed_tokens, self.gain_value):+ # sum_bg = sum([1 if x == bg else 0 for x in tokens[i]])+ # sum_ed = sum([1 if x == ed else 0 for x in tokens[i]])+ # logits[i, bg] += ga+ # if sum_bg > sum_ed or tokens[i,-1] in [bg, ed]:+ # logits[i, bg] = -np.inf+ # if sum_bg <= sum_ed:+ # logits[i, ed] = -np.inf+++class ApplyTimestampRules(LogitFilter):+ def __init__(+ self,+ tokenizer: Tokenizer,+ sample_begin: int,+ max_initial_timestamp_index: Optional[int],+ ):+ self.tokenizer = tokenizer+ self.sample_begin = sample_begin+ self.max_initial_timestamp_index = max_initial_timestamp_index++ def apply(self, logits: Tensor, tokens: Tensor):+ # suppress <|notimestamps|> which is handled by without_timestamps+ if self.tokenizer.no_timestamps is not None:+ logits[:, self.tokenizer.no_timestamps] = -np.inf++ # timestamps have to appear in pairs, except directly before EOT; mask logits accordingly+ for k in range(tokens.shape[0]):+ sampled_tokens = tokens[k, self.sample_begin :]+ seq = [t for t in sampled_tokens.tolist()]+ last_was_timestamp = len(seq) >= 1 and seq[-1] >= self.tokenizer.timestamp_begin+ penultimate_was_timestamp = len(seq) < 2 or seq[-2] >= self.tokenizer.timestamp_begin++ if last_was_timestamp:+ if penultimate_was_timestamp: # has to be non-timestamp+ logits[k, self.tokenizer.timestamp_begin :] = -np.inf+ else: # cannot be normal text tokens+ logits[k, : self.tokenizer.eot] = -np.inf++ timestamps = sampled_tokens[sampled_tokens.ge(self.tokenizer.timestamp_begin)]+ if timestamps.numel() > 0:+ # timestamps shouldn't decrease; forbid timestamp tokens smaller than the last+ # also force each segment to have a nonzero length, to prevent infinite looping+ if last_was_timestamp and not penultimate_was_timestamp:+ timestamp_last = timestamps[-1]+ else:+ timestamp_last = timestamps[-1] + 1+ logits[k, self.tokenizer.timestamp_begin : timestamp_last] = -np.inf++ if tokens.shape[1] == self.sample_begin:+ # suppress generating non-timestamp tokens at the beginning+ logits[:, : self.tokenizer.timestamp_begin] = -np.inf++ # apply the `max_initial_timestamp` option+ if self.max_initial_timestamp_index is not None:+ last_allowed = self.tokenizer.timestamp_begin + self.max_initial_timestamp_index+ logits[:, last_allowed + 1 :] = -np.inf++ # if sum of probability over timestamps is above any other token, sample timestamp+ logprobs = F.log_softmax(logits.float(), dim=-1)+ for k in range(tokens.shape[0]):+ timestamp_logprob = logprobs[k, self.tokenizer.timestamp_begin :].logsumexp(dim=-1)+ max_text_token_logprob = logprobs[k, : self.tokenizer.timestamp_begin].max()+ if timestamp_logprob > max_text_token_logprob:+ logits[k, : self.tokenizer.timestamp_begin] = -np.inf+++class DecodingTask:+ inference: Inference+ sequence_ranker: SequenceRanker+ decoder: TokenDecoder+ logit_filters: List[LogitFilter]++ def __init__(self, model: "Whisper", options: DecodingOptions):+ self.model = model++ language = options.language or "en"+ tokenizer = get_tokenizer(+ model.is_multilingual,+ num_languages=model.num_languages,+ language=language,+ task=options.task,+ vocab_path=options.vocab_path,+ )+ self.tokenizer: Tokenizer = tokenizer+ self.options: DecodingOptions = self._verify_options(options)++ self.n_group: int = options.beam_size or options.best_of or 1+ self.n_ctx: int = model.dims.n_text_ctx+ self.sample_len: int = options.sample_len or model.dims.n_text_ctx // 2++ self.sot_sequence: Tuple[int] = tokenizer.sot_sequence+ if self.options.without_timestamps:+ self.sot_sequence = tokenizer.sot_sequence_including_notimestamps++ self.initial_tokens: Tuple[int] = self._get_initial_tokens()+ self.sample_begin: int = len(self.initial_tokens)+ self.sot_index: int = self.initial_tokens.index(tokenizer.sot)++ # inference: implements the forward pass through the decoder, including kv caching+ self.inference = PyTorchInference(model, len(self.initial_tokens))++ # sequence ranker: implements how to rank a group of sampled sequences+ self.sequence_ranker = MaximumLikelihoodRanker(options.length_penalty)++ # decoder: implements how to select the next tokens, given the autoregressive distribution+ if options.beam_size is not None:+ self.decoder = BeamSearchDecoder(+ options.beam_size, tokenizer.eot, self.inference, options.patience+ )+ else:+ self.decoder = GreedyDecoder(options.temperature, tokenizer.eot)++ # logit filters: applies various rules to suppress or penalize certain tokens+ self.logit_filters = []+ if self.options.suppress_blank:+ self.logit_filters.append(SuppressBlank(self.tokenizer, self.sample_begin))+ if self.options.suppress_tokens:+ self.logit_filters.append(SuppressTokens(self._get_suppress_tokens()))+ if self.options.gain_event:+ self.logit_filters.append(+ GainEventToken(+ self.tokenizer.encode(self.options.gain_tokens_bg, allowed_special="all"),+ self.tokenizer.encode(self.options.gain_tokens_ed, allowed_special="all"),+ self.options.gain_tokens_score,+ )+ )+ if self.options.use_emo_threshold:+ self.logit_filters.append(+ ThresholdEmoToken(+ self.tokenizer.encode(self.options.emo_unk_token, allowed_special="all"),+ self.tokenizer.encode(self.options.emo_target_tokens, allowed_special="all"),+ self.options.emo_target_threshold,+ )+ )+ if not options.without_timestamps:+ precision = CHUNK_LENGTH / model.dims.n_audio_ctx # usually 0.02 seconds+ max_initial_timestamp_index = None+ if options.max_initial_timestamp:+ max_initial_timestamp_index = round(self.options.max_initial_timestamp / precision)+ self.logit_filters.append(+ ApplyTimestampRules(tokenizer, self.sample_begin, max_initial_timestamp_index)+ )++ def _verify_options(self, options: DecodingOptions) -> DecodingOptions:+ if options.beam_size is not None and options.best_of is not None:+ raise ValueError("beam_size and best_of can't be given together")+ if options.temperature == 0:+ if options.best_of is not None:+ raise ValueError("best_of with greedy sampling (T=0) is not compatible")+ if options.patience is not None and options.beam_size is None:+ raise ValueError("patience requires beam_size to be given")+ if options.length_penalty is not None and not (0 <= options.length_penalty <= 1):+ raise ValueError("length_penalty (alpha) should be a value between 0 and 1")++ return options++ def _get_initial_tokens(self) -> Tuple[int]:+ tokens = list(self.sot_sequence)++ if prefix := self.options.prefix:+ prefix_tokens = (+ self.tokenizer.encode(" " + prefix.strip()) if isinstance(prefix, str) else prefix+ )+ if self.sample_len is not None:+ max_prefix_len = self.n_ctx // 2 - self.sample_len+ prefix_tokens = prefix_tokens[-max_prefix_len:]+ tokens = tokens + prefix_tokens++ if prompt := self.options.prompt:+ prompt_tokens = (+ self.tokenizer.encode(" " + prompt.strip()) if isinstance(prompt, str) else prompt+ )+ tokens = [self.tokenizer.sot_prev] + prompt_tokens[-(self.n_ctx // 2 - 1) :] + tokens+ # FIX(funasr): sense vocie+ if initial_prompt := self.options.initial_prompt:+ if self.options.language is not None:+ initial_prompt = f"{initial_prompt}<|{self.options.language}|>"+ tokens = self.tokenizer.encode(initial_prompt, allowed_special="all")+ else:+ tokens = self.tokenizer.encode(initial_prompt, allowed_special="all")+ tokens += [0]++ return tuple(tokens)++ def _get_suppress_tokens(self) -> Tuple[int]:+ suppress_tokens = self.options.suppress_tokens++ if isinstance(suppress_tokens, str):+ suppress_tokens = [int(t) for t in suppress_tokens.split(",")]++ if -1 in suppress_tokens:+ suppress_tokens = [t for t in suppress_tokens if t >= 0]+ suppress_tokens.extend(self.tokenizer.non_speech_tokens)+ elif suppress_tokens is None or len(suppress_tokens) == 0:+ suppress_tokens = [] # interpret empty string as an empty list+ else:+ assert isinstance(suppress_tokens, list), "suppress_tokens must be a list"++ suppress_tokens.extend(+ [+ self.tokenizer.transcribe,+ self.tokenizer.translate,+ self.tokenizer.sot,+ self.tokenizer.sot_prev,+ self.tokenizer.sot_lm,+ ]+ )+ if self.tokenizer.no_speech is not None:+ # no-speech probability is collected separately+ suppress_tokens.append(self.tokenizer.no_speech)++ return tuple(sorted(set(suppress_tokens)))++ def _get_audio_features(self, mel: Tensor):+ if self.options.fp16:+ mel = mel.half()++ if mel.shape[-2:] == (+ self.model.dims.n_audio_ctx,+ self.model.dims.n_audio_state,+ ):+ # encoded audio features are given; skip audio encoding+ audio_features = mel+ else:+ audio_features = self.model.encoder(mel)+ if isinstance(audio_features, tuple):+ audio_features = audio_features[0]++ if audio_features.dtype != (torch.float16 if self.options.fp16 else torch.float32):+ return TypeError(f"audio_features has an incorrect dtype: {audio_features.dtype}")++ return audio_features++ def _detect_language(self, audio_features: Tensor, tokens: Tensor):+ languages = [self.options.language] * audio_features.shape[0]+ lang_probs = None++ if self.options.language is None or self.options.task == "lang_id":+ lang_tokens, lang_probs = self.model.detect_language(+ audio_features, self.tokenizer, x=tokens+ )+ languages = [max(probs, key=probs.get) for probs in lang_probs]+ # FIX(funasr): sense vocie+ # if self.options.language is None:+ # tokens[:, self.sot_index + 1] = lang_tokens # write language tokens+ if self.options.language is None:+ # tokens[:, self.sot_index + 1] = lang_tokens # write language tokens+ languages = "".join([f"<|{language}|>" for language in languages])++ n_audio = audio_features.shape[0]+ lang_tokens = torch.tensor(+ [self.tokenizer.encode(languages, allowed_special="all")] * n_audio+ ).to(+ audio_features.device+ ) # [n_audio, 1]++ tokens[:, -1:] = lang_tokens[:, :]+ languages = [languages]++ return languages, lang_probs++ def _main_loop(self, audio_features: Tensor, tokens: Tensor):+ n_batch = tokens.shape[0]+ sum_logprobs: Tensor = torch.zeros(n_batch, device=audio_features.device)+ no_speech_probs = [np.nan] * n_batch++ try:+ for i in range(self.sample_len):+ logits = self.inference.logits(tokens, audio_features)++ if i == 0 and self.tokenizer.no_speech is not None: # save no_speech_probs+ probs_at_sot = logits[:, self.sot_index].float().softmax(dim=-1)+ no_speech_probs = probs_at_sot[:, self.tokenizer.no_speech].tolist()++ # now we need to consider the logits at the last token only+ logits = logits[:, -1]++ # apply the logit filters, e.g. for suppressing or applying penalty to+ for logit_filter in self.logit_filters:+ logit_filter.apply(logits, tokens)++ # expand the tokens tensor with the selected next tokens+ tokens, completed = self.decoder.update(tokens, logits, sum_logprobs)++ if completed or tokens.shape[-1] > self.n_ctx:+ break+ finally:+ self.inference.cleanup_caching()++ return tokens, sum_logprobs, no_speech_probs++ @torch.no_grad()+ def run(self, mel: Tensor) -> List[DecodingResult]:+ self.decoder.reset()+ tokenizer: Tokenizer = self.tokenizer+ n_audio: int = mel.shape[0]++ audio_features: Tensor = self._get_audio_features(mel) # encoder forward pass+ tokens: Tensor = torch.tensor([self.initial_tokens]).repeat(n_audio, 1)++ # detect language if requested, overwriting the language token+ languages, language_probs = self._detect_language(audio_features, tokens)+ if self.options.task == "lang_id":+ return [+ DecodingResult(audio_features=features, language=language, language_probs=probs)+ for features, language, probs in zip(audio_features, languages, language_probs)+ ]++ # repeat text tensors by the group size, for beam search or best-of-n sampling+ tokens = tokens.repeat_interleave(self.n_group, dim=0).to(audio_features.device)++ # call the main sampling loop+ tokens, sum_logprobs, no_speech_probs = self._main_loop(audio_features, tokens)++ # reshape the tensors to have (n_audio, n_group) as the first two dimensions+ audio_features = audio_features[:: self.n_group]+ no_speech_probs = no_speech_probs[:: self.n_group]+ assert audio_features.shape[0] == len(no_speech_probs) == n_audio++ tokens = tokens.reshape(n_audio, self.n_group, -1)+ sum_logprobs = sum_logprobs.reshape(n_audio, self.n_group)++ # get the final candidates for each group, and slice between the first sampled token and EOT+ tokens, sum_logprobs = self.decoder.finalize(tokens, sum_logprobs)+ tokens: List[List[Tensor]] = [+ [t[self.sample_begin : (t == tokenizer.eot).nonzero()[0, 0]] for t in s] for s in tokens+ ]++ # select the top-ranked sample in each group+ selected = self.sequence_ranker.rank(tokens, sum_logprobs)+ tokens: List[List[int]] = [t[i].tolist() for i, t in zip(selected, tokens)]+ texts: List[str] = [tokenizer.decode(t).strip() for t in tokens]++ sum_logprobs: List[float] = [lp[i] for i, lp in zip(selected, sum_logprobs)]+ avg_logprobs: List[float] = [lp / (len(t) + 1) for t, lp in zip(tokens, sum_logprobs)]++ fields = (+ texts,+ languages,+ tokens,+ audio_features,+ avg_logprobs,+ no_speech_probs,+ )+ if len(set(map(len, fields))) != 1:+ raise RuntimeError(f"inconsistent result lengths: {list(map(len, fields))}")++ return [+ DecodingResult(+ audio_features=features,+ language=language,+ tokens=tokens,+ text=text,+ avg_logprob=avg_logprob,+ no_speech_prob=no_speech_prob,+ temperature=self.options.temperature,+ compression_ratio=compression_ratio(text),+ )+ for text, language, tokens, features, avg_logprob, no_speech_prob in zip(*fields)+ ]++[email protected]_grad()+def decode(+ model: "Whisper",+ mel: Tensor,+ options: DecodingOptions = DecodingOptions(),+ **kwargs,+) -> Union[DecodingResult, List[DecodingResult]]:+ """+ Performs decoding of 30-second audio segment(s), provided as Mel spectrogram(s).++ Parameters+ ----------+ model: Whisper+ the Whisper model instance++ mel: torch.Tensor, shape = (80, 3000) or (*, 80, 3000)+ A tensor containing the Mel spectrogram(s)++ options: DecodingOptions+ A dataclass that contains all necessary options for decoding 30-second segments++ Returns+ -------+ result: Union[DecodingResult, List[DecodingResult]]+ The result(s) of decoding contained in `DecodingResult` dataclass instance(s)+ """+ if single := mel.ndim == 2:+ mel = mel.unsqueeze(0)++ if kwargs:+ options = replace(options, **kwargs)++ result = DecodingTask(model, options).run(mel)++ return result[0] if single else resultdiff --git a/funasr/models/sense_voice/whisper_lib/model.py b/funasr/models/sense_voice/whisper_lib/model.pynew file mode 100644index 000000000..eb1380d11--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/model.py@@ -0,0 +1,459 @@+import base64+import gzip+from dataclasses import dataclass+from typing import Dict, Iterable, Optional++import numpy as np+import torch+import torch.nn.functional as F+from torch import Tensor, nn++from .decoding import decode as decode_function+from .decoding import detect_language as detect_language_function+from .transcribe import transcribe as transcribe_function+++@dataclass+class ModelDimensions:+ n_mels: int+ n_audio_ctx: int+ n_audio_state: int+ n_audio_head: int+ n_audio_layer: int+ n_vocab: int+ n_text_ctx: int+ n_text_state: int+ n_text_head: int+ n_text_layer: int+ att_type: str = "default"+++# class LayerNorm(nn.LayerNorm):+# def forward(self, x: Tensor) -> Tensor:+# return super().forward(x.float()).type(x.dtype)+++class LayerNorm(nn.LayerNorm):+ def __init__(self, *args, **kwargs):+ super().__init__(*args, **kwargs)++ def forward(self, input):+ output = F.layer_norm(+ input.float(),+ self.normalized_shape,+ self.weight.float() if self.weight is not None else None,+ self.bias.float() if self.bias is not None else None,+ self.eps,+ )+ return output.type_as(input)+++class Linear(nn.Linear):+ def forward(self, x: Tensor) -> Tensor:+ return F.linear(+ x,+ self.weight.to(x.dtype),+ None if self.bias is None else self.bias.to(x.dtype),+ )+++class Conv1d(nn.Conv1d):+ def _conv_forward(self, x: Tensor, weight: Tensor, bias: Optional[Tensor]) -> Tensor:+ return super()._conv_forward(+ x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype)+ )+++def sinusoids(length, channels, max_timescale=10000):+ """Returns sinusoids for positional embedding"""+ assert channels % 2 == 0+ log_timescale_increment = np.log(max_timescale) / (channels // 2 - 1)+ inv_timescales = torch.exp(-log_timescale_increment * torch.arange(channels // 2))+ scaled_time = torch.arange(length)[:, np.newaxis] * inv_timescales[np.newaxis, :]+ return torch.cat([torch.sin(scaled_time), torch.cos(scaled_time)], dim=1)+++class MultiHeadAttention(nn.Module):+ def __init__(self, n_state: int, n_head: int):+ super().__init__()+ self.n_head = n_head+ self.query = Linear(n_state, n_state)+ self.key = Linear(n_state, n_state, bias=False)+ self.value = Linear(n_state, n_state)+ self.out = Linear(n_state, n_state)++ def forward(+ self,+ x: Tensor,+ xa: Optional[Tensor] = None,+ mask: Optional[Tensor] = None,+ kv_cache: Optional[dict] = None,+ **kwargs,+ ):+ is_pad_mask = kwargs.get("is_pad_mask", False)++ q = self.query(x)++ if kv_cache is None or xa is None or self.key not in kv_cache:+ # hooks, if installed (i.e. kv_cache is not None), will prepend the cached kv tensors;+ # otherwise, perform key/value projections for self- or cross-attention as usual.+ k = self.key(x if xa is None else xa)+ v = self.value(x if xa is None else xa)+ else:+ # for cross-attention, calculate keys and values once and reuse in subsequent calls.+ k = kv_cache[self.key]+ v = kv_cache[self.value]++ wv, qk = self.qkv_attention(q, k, v, mask, is_pad_mask=is_pad_mask)+ return self.out(wv), qk++ def qkv_attention(+ self,+ q: Tensor,+ k: Tensor,+ v: Tensor,+ mask: Optional[Tensor] = None,+ **kwargs,+ ):+ is_pad_mask = kwargs.get("is_pad_mask", False)+ n_batch, n_ctx, n_state = q.shape+ scale = (n_state // self.n_head) ** -0.25+ q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) * scale+ k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 3, 1) * scale+ v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)++ qk = q @ k+ if mask is not None:+ if not is_pad_mask:+ qk = qk + mask[:n_ctx, :n_ctx]+ else:+ mask = mask.unsqueeze(1).eq(0) # (batch, 1, t, 1)+ min_value = -float(+ "inf"+ ) # min_value = float(np.finfo(torch.tensor(0, dtype=qk.dtype).numpy().dtype).min)+ qk = qk.masked_fill(mask, min_value)++ qk = qk.float()++ w = F.softmax(qk, dim=-1).to(q.dtype)+ if mask is not None and is_pad_mask:+ w = w.masked_fill(mask, 0.0)+ return (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2), qk.detach()+++class MultiHeadAttentionSdpa(nn.Module):+ def __init__(self, n_state: int, n_head: int):+ super().__init__()+ self.n_head = n_head+ self.query = Linear(n_state, n_state)+ self.key = Linear(n_state, n_state, bias=False)+ self.value = Linear(n_state, n_state)+ self.out = Linear(n_state, n_state)++ def forward(+ self,+ x: Tensor,+ xa: Optional[Tensor] = None,+ mask: Optional[Tensor] = None,+ kv_cache: Optional[dict] = None,+ **kwargs,+ ):+ is_pad_mask = kwargs.get("is_pad_mask", False)++ q = self.query(x)++ if kv_cache is None or xa is None or self.key not in kv_cache:+ # hooks, if installed (i.e. kv_cache is not None), will prepend the cached kv tensors;+ # otherwise, perform key/value projections for self- or cross-attention as usual.+ k = self.key(x if xa is None else xa)+ v = self.value(x if xa is None else xa)+ else:+ # for cross-attention, calculate keys and values once and reuse in subsequent calls.+ k = kv_cache[self.key]+ v = kv_cache[self.value]++ wv, qk = self.qkv_attention(q, k, v, mask, is_pad_mask=is_pad_mask, is_causal=False)+ return self.out(wv), qk++ def qkv_attention(+ self,+ q: Tensor,+ k: Tensor,+ v: Tensor,+ mask: Optional[Tensor] = None,+ **kwargs,+ ):+ is_pad_mask = kwargs.get("is_pad_mask", False)+ is_causal = kwargs.get("is_causal", False)+ n_batch, n_ctx, n_state = q.shape+ scale = (n_state // self.n_head) ** -0.5+ q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)+ k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)+ v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)++ if mask is not None:+ if not is_pad_mask:+ mask = None+ is_causal = True+ else:+ mask = mask.unsqueeze(1).to(torch.bool) # (batch, 1, 1, t)++ attn_output = torch.nn.functional.scaled_dot_product_attention(+ q,+ k,+ v,+ attn_mask=mask,+ dropout_p=0.0,+ is_causal=is_causal,+ scale=scale,+ )+ if mask is not None:+ attn_output = attn_output.masked_fill(mask.transpose(2, 3).logical_not(), 0.0)+ attn_output = attn_output.transpose(1, 2)+ attn_output = attn_output.flatten(start_dim=2)+ return attn_output, None+++att_type_dict = {+ "default": MultiHeadAttention,+ "sdpa": MultiHeadAttentionSdpa,+}+++class ResidualAttentionBlock(nn.Module):+ def __init__(self, n_state: int, n_head: int, cross_attention: bool = False, **kwargs):+ super().__init__()++ att_type = kwargs.get("att_type", "default")+ self.attn = att_type_dict[att_type](n_state, n_head) # MultiHeadAttention(n_state, n_head)+ self.attn_ln = LayerNorm(n_state)++ self.cross_attn = (+ att_type_dict[att_type](n_state, n_head) if cross_attention else None+ ) # MultiHeadAttention(n_state, n_head) if cross_attention else None+ self.cross_attn_ln = LayerNorm(n_state) if cross_attention else None++ n_mlp = n_state * 4+ self.mlp = nn.Sequential(Linear(n_state, n_mlp), nn.GELU(), Linear(n_mlp, n_state))+ self.mlp_ln = LayerNorm(n_state)++ def forward(+ self,+ x: Tensor,+ xa: Optional[Tensor] = None,+ mask: Optional[Tensor] = None,+ kv_cache: Optional[dict] = None,+ **kwargs,+ ):+ is_pad_mask = kwargs.get("is_pad_mask", False)+ is_pad_memory_mask = kwargs.get("is_pad_memory_mask", False)+ memory_mask = kwargs.get("memory_mask", None)+ x = x + self.attn(self.attn_ln(x), mask=mask, kv_cache=kv_cache, is_pad_mask=is_pad_mask)[0]+ if self.cross_attn:+ x = (+ x+ + self.cross_attn(+ self.cross_attn_ln(x),+ xa,+ mask=memory_mask,+ kv_cache=kv_cache,+ is_pad_mask=is_pad_memory_mask,+ )[0]+ )+ x = x + self.mlp(self.mlp_ln(x))+ return x+++class AudioEncoder(nn.Module):+ def __init__(self, n_mels: int, n_ctx: int, n_state: int, n_head: int, n_layer: int, **kwargs):+ super().__init__()+ self.conv1 = Conv1d(n_mels, n_state, kernel_size=3, stride=2, padding=1)+ self.conv2 = Conv1d(n_state, n_state, kernel_size=3, stride=2, padding=1)+ self.register_buffer("positional_embedding", sinusoids(n_ctx, n_state))++ self.blocks: Iterable[ResidualAttentionBlock] = nn.ModuleList(+ [+ ResidualAttentionBlock(n_state, n_head, att_type=kwargs.get("att_type", "default"))+ for _ in range(n_layer)+ ]+ )+ self.ln_post = LayerNorm(n_state)++ def forward(self, x: Tensor):+ """+ x : torch.Tensor, shape = (batch_size, n_mels, n_ctx)+ the mel spectrogram of the audio+ """+ x = F.gelu(self.conv1(x))+ x = F.gelu(self.conv2(x))+ x = x.permute(0, 2, 1)++ # assert x.shape[1:] == self.positional_embedding.shape, "incorrect audio shape"+ # x = (x + self.positional_embedding).to(x.dtype)+ x = (x + self.positional_embedding[: x.size(1), :]).to(x.dtype)++ for block in self.blocks:+ x = block(x)++ x = self.ln_post(x)+ return x+++class TextDecoder(nn.Module):+ def __init__(self, n_vocab: int, n_ctx: int, n_state: int, n_head: int, n_layer: int, **kwargs):+ super().__init__()++ self.token_embedding = nn.Embedding(n_vocab, n_state)+ self.positional_embedding = nn.Parameter(torch.empty(n_ctx, n_state))++ self.blocks: Iterable[ResidualAttentionBlock] = nn.ModuleList(+ [+ ResidualAttentionBlock(+ n_state,+ n_head,+ cross_attention=True,+ att_type="default", # kwargs.get("att_type", "default"),+ )+ for _ in range(n_layer)+ ]+ )+ self.ln = LayerNorm(n_state)++ mask = torch.empty(n_ctx, n_ctx).fill_(-np.inf).triu_(1)+ self.register_buffer("mask", mask, persistent=False)++ def forward(self, x: Tensor, xa: Tensor, kv_cache: Optional[dict] = None):+ """+ x : torch.LongTensor, shape = (batch_size, <= n_ctx)+ the text tokens+ xa : torch.Tensor, shape = (batch_size, n_audio_ctx, n_audio_state)+ the encoded audio features to be attended on+ """+ offset = next(iter(kv_cache.values())).shape[1] if kv_cache else 0+ x = self.token_embedding(x) + self.positional_embedding[offset : offset + x.shape[-1]]+ x = x.to(xa.dtype)++ for block in self.blocks:+ x = block(x, xa, mask=self.mask, kv_cache=kv_cache)++ x = self.ln(x)+ logits = (x @ torch.transpose(self.token_embedding.weight.to(x.dtype), 0, 1)).float()++ return logits++ def init_state(self, x):+ state = {}++ return state++ def final_score(self, state) -> float:+ """Score eos (optional).++ Args:+ state: Scorer state for prefix tokens++ Returns:+ float: final score++ """+ return 0.0++ def score(self, ys, state, x):+ """Score."""+ # ys_mask = subsequent_mask(len(ys), device=x.device).unsqueeze(0)+ # print(ys.unsqueeze(0).size())+ logp = self.forward(ys, x.unsqueeze(0), cache=state)+ logp = torch.log_softmax(logp, dim=-1)+ return logp[:, -1, :], state+++class Whisper(nn.Module):+ def __init__(self, dims: ModelDimensions):+ super().__init__()+ self.dims = dims+ self.encoder = AudioEncoder(+ self.dims.n_mels,+ self.dims.n_audio_ctx,+ self.dims.n_audio_state,+ self.dims.n_audio_head,+ self.dims.n_audio_layer,+ att_type=self.dims.att_type,+ )+ self.decoder = TextDecoder(+ self.dims.n_vocab,+ self.dims.n_text_ctx,+ self.dims.n_text_state,+ self.dims.n_text_head,+ self.dims.n_text_layer,+ att_type=self.dims.att_type,+ )+ # use the last half among the decoder layers for time alignment by default;+ # to use a specific set of heads, see `set_alignment_heads()` below.+ all_heads = torch.zeros(self.dims.n_text_layer, self.dims.n_text_head, dtype=torch.bool)+ all_heads[self.dims.n_text_layer // 2 :] = True+ # self.register_buffer("alignment_heads", all_heads.to_sparse(), persistent=False)+ # alignment_heads_dense = model.get_buffer("alignment_heads").to_dense()+ # model.register_buffer("alignment_heads", alignment_heads_dense, persistent=False)++ def set_alignment_heads(self, dump: bytes):+ array = np.frombuffer(gzip.decompress(base64.b85decode(dump)), dtype=bool).copy()+ mask = torch.from_numpy(array).reshape(self.dims.n_text_layer, self.dims.n_text_head)+ self.register_buffer("alignment_heads", mask.to_sparse(), persistent=False)++ def embed_audio(self, mel: torch.Tensor):+ return self.encoder(mel)++ def logits(self, tokens: torch.Tensor, audio_features: torch.Tensor):+ return self.decoder(tokens, audio_features)++ def forward(self, mel: torch.Tensor, tokens: torch.Tensor) -> Dict[str, torch.Tensor]:+ return self.decoder(tokens, self.encoder(mel))++ @property+ def device(self):+ return next(self.parameters()).device++ @property+ def is_multilingual(self):+ return self.dims.n_vocab >= 51865++ @property+ def num_languages(self):+ return self.dims.n_vocab - 51765 - int(self.is_multilingual)++ def install_kv_cache_hooks(self, cache: Optional[dict] = None):+ """+ The `MultiHeadAttention` module optionally accepts `kv_cache` which stores the key and value+ tensors calculated for the previous positions. This method returns a dictionary that stores+ all caches, and the necessary hooks for the key and value projection modules that save the+ intermediate tensors to be reused during later calculations.++ Returns+ -------+ cache : Dict[nn.Module, torch.Tensor]+ A dictionary object mapping the key/value projection modules to its cache+ hooks : List[RemovableHandle]+ List of PyTorch RemovableHandle objects to stop the hooks to be called+ """+ cache = {**cache} if cache is not None else {}+ hooks = []++ def save_to_cache(module, _, output):+ if module not in cache or output.shape[1] > self.dims.n_text_ctx:+ # save as-is, for the first token or cross attention+ cache[module] = output+ else:+ cache[module] = torch.cat([cache[module], output], dim=1).detach()+ return cache[module]++ def install_hooks(layer: nn.Module):+ if isinstance(layer, MultiHeadAttention):+ hooks.append(layer.key.register_forward_hook(save_to_cache))+ hooks.append(layer.value.register_forward_hook(save_to_cache))++ self.decoder.apply(install_hooks)+ return cache, hooks++ detect_language = detect_language_function+ transcribe = transcribe_function+ decode = decode_functiondiff --git a/funasr/models/sense_voice/whisper_lib/normalizers/__init__.py b/funasr/models/sense_voice/whisper_lib/normalizers/__init__.pynew file mode 100644index 000000000..896d5e336--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/normalizers/__init__.py@@ -0,0 +1,2 @@+from .basic import BasicTextNormalizer as BasicTextNormalizer+from .english import EnglishTextNormalizer as EnglishTextNormalizerdiff --git a/funasr/models/sense_voice/whisper_lib/normalizers/basic.py b/funasr/models/sense_voice/whisper_lib/normalizers/basic.pynew file mode 100644index 000000000..76addc520--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/normalizers/basic.py@@ -0,0 +1,75 @@+import re+import unicodedata++import regex++# non-ASCII letters that are not separated by "NFKD" normalization+ADDITIONAL_DIACRITICS = {+ "œ": "oe",+ "Œ": "OE",+ "ø": "o",+ "Ø": "O",+ "æ": "ae",+ "Æ": "AE",+ "ß": "ss",+ "ẞ": "SS",+ "đ": "d",+ "Đ": "D",+ "ð": "d",+ "Ð": "D",+ "þ": "th",+ "Þ": "th",+ "ł": "l",+ "Ł": "L",+}+++def remove_symbols_and_diacritics(s: str, keep=""):+ """+ Replace any other markers, symbols, and punctuations with a space,+ and drop any diacritics (category 'Mn' and some manual mappings)+ """+ return "".join(+ (+ c+ if c in keep+ else (+ ADDITIONAL_DIACRITICS[c]+ if c in ADDITIONAL_DIACRITICS+ else (+ ""+ if unicodedata.category(c) == "Mn"+ else " " if unicodedata.category(c)[0] in "MSP" else c+ )+ )+ )+ for c in unicodedata.normalize("NFKD", s)+ )+++def remove_symbols(s: str):+ """+ Replace any other markers, symbols, punctuations with a space, keeping diacritics+ """+ return "".join(+ " " if unicodedata.category(c)[0] in "MSP" else c for c in unicodedata.normalize("NFKC", s)+ )+++class BasicTextNormalizer:+ def __init__(self, remove_diacritics: bool = False, split_letters: bool = False):+ self.clean = remove_symbols_and_diacritics if remove_diacritics else remove_symbols+ self.split_letters = split_letters++ def __call__(self, s: str):+ s = s.lower()+ s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets+ s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis+ s = self.clean(s).lower()++ if self.split_letters:+ s = " ".join(regex.findall(r"\X", s, regex.U))++ s = re.sub(r"\s+", " ", s) # replace any successive whitespace characters with a space++ return sdiff --git a/funasr/models/sense_voice/whisper_lib/normalizers/english.json b/funasr/models/sense_voice/whisper_lib/normalizers/english.jsonnew file mode 100644index 000000000..74a1c3521--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/normalizers/english.json@@ -0,0 +1,1741 @@+{+ "accessorise": "accessorize",+ "accessorised": "accessorized",+ "accessorises": "accessorizes",+ "accessorising": "accessorizing",+ "acclimatisation": "acclimatization",+ "acclimatise": "acclimatize",+ "acclimatised": "acclimatized",+ "acclimatises": "acclimatizes",+ "acclimatising": "acclimatizing",+ "accoutrements": "accouterments",+ "aeon": "eon",+ "aeons": "eons",+ "aerogramme": "aerogram",+ "aerogrammes": "aerograms",+ "aeroplane": "airplane",+ "aeroplanes": "airplanes",+ "aesthete": "esthete",+ "aesthetes": "esthetes",+ "aesthetic": "esthetic",+ "aesthetically": "esthetically",+ "aesthetics": "esthetics",+ "aetiology": "etiology",+ "ageing": "aging",+ "aggrandisement": "aggrandizement",+ "agonise": "agonize",+ "agonised": "agonized",+ "agonises": "agonizes",+ "agonising": "agonizing",+ "agonisingly": "agonizingly",+ "almanack": "almanac",+ "almanacks": "almanacs",+ "aluminium": "aluminum",+ "amortisable": "amortizable",+ "amortisation": "amortization",+ "amortisations": "amortizations",+ "amortise": "amortize",+ "amortised": "amortized",+ "amortises": "amortizes",+ "amortising": "amortizing",+ "amphitheatre": "amphitheater",+ "amphitheatres": "amphitheaters",+ "anaemia": "anemia",+ "anaemic": "anemic",+ "anaesthesia": "anesthesia",+ "anaesthetic": "anesthetic",+ "anaesthetics": "anesthetics",+ "anaesthetise": "anesthetize",+ "anaesthetised": "anesthetized",+ "anaesthetises": "anesthetizes",+ "anaesthetising": "anesthetizing",+ "anaesthetist": "anesthetist",+ "anaesthetists": "anesthetists",+ "anaesthetize": "anesthetize",+ "anaesthetized": "anesthetized",+ "anaesthetizes": "anesthetizes",+ "anaesthetizing": "anesthetizing",+ "analogue": "analog",+ "analogues": "analogs",+ "analyse": "analyze",+ "analysed": "analyzed",+ "analyses": "analyzes",+ "analysing": "analyzing",+ "anglicise": "anglicize",+ "anglicised": "anglicized",+ "anglicises": "anglicizes",+ "anglicising": "anglicizing",+ "annualised": "annualized",+ "antagonise": "antagonize",+ "antagonised": "antagonized",+ "antagonises": "antagonizes",+ "antagonising": "antagonizing",+ "apologise": "apologize",+ "apologised": "apologized",+ "apologises": "apologizes",+ "apologising": "apologizing",+ "appal": "appall",+ "appals": "appalls",+ "appetiser": "appetizer",+ "appetisers": "appetizers",+ "appetising": "appetizing",+ "appetisingly": "appetizingly",+ "arbour": "arbor",+ "arbours": "arbors",+ "archeological": "archaeological",+ "archaeologically": "archeologically",+ "archaeologist": "archeologist",+ "archaeologists": "archeologists",+ "archaeology": "archeology</span>",+ "ardour": "ardor",+ "armour": "armor",+ "armoured": "armored",+ "armourer": "armorer",+ "armourers": "armorers",+ "armouries": "armories",+ "armoury": "armory",+ "artefact": "artifact",+ "artefacts": "artifacts",+ "authorise": "authorize",+ "authorised": "authorized",+ "authorises": "authorizes",+ "authorising": "authorizing",+ "axe": "ax",+ "backpedalled": "backpedaled",+ "backpedalling": "backpedaling",+ "bannister": "banister",+ "bannisters": "banisters",+ "baptise": "baptize",+ "baptised": "baptized",+ "baptises": "baptizes",+ "baptising": "baptizing",+ "bastardise": "bastardize",+ "bastardised": "bastardized",+ "bastardises": "bastardizes",+ "bastardising": "bastardizing",+ "battleax": "battleaxe",+ "baulk": "balk",+ "baulked": "balked",+ "baulking": "balking",+ "baulks": "balks",+ "bedevilled": "bedeviled",+ "bedevilling": "bedeviling",+ "behaviour": "behavior",+ "behavioural": "behavioral",+ "behaviourism": "behaviorism",+ "behaviourist": "behaviorist",+ "behaviourists": "behaviorists",+ "behaviours": "behaviors",+ "behove": "behoove",+ "behoved": "behooved",+ "behoves": "behooves",+ "bejewelled": "bejeweled",+ "belabour": "belabor",+ "belaboured": "belabored",+ "belabouring": "belaboring",+ "belabours": "belabors",+ "bevelled": "beveled",+ "bevvies": "bevies",+ "bevvy": "bevy",+ "biassed": "biased",+ "biassing": "biasing",+ "bingeing": "binging",+ "bougainvillaea": "bougainvillea",+ "bougainvillaeas": "bougainvilleas",+ "bowdlerise": "bowdlerize",+ "bowdlerised": "bowdlerized",+ "bowdlerises": "bowdlerizes",+ "bowdlerising": "bowdlerizing",+ "breathalyse": "breathalyze",+ "breathalysed": "breathalyzed",+ "breathalyser": "breathalyzer",+ "breathalysers": "breathalyzers",+ "breathalyses": "breathalyzes",+ "breathalysing": "breathalyzing",+ "brutalise": "brutalize",+ "brutalised": "brutalized",+ "brutalises": "brutalizes",+ "brutalising": "brutalizing",+ "busses": "buses",+ "bussing": "busing",+ "caesarean": "cesarean",+ "caesareans": "cesareans",+ "calibre": "caliber",+ "calibres": "calibers",+ "calliper": "caliper",+ "callipers": "calipers",+ "callisthenics": "calisthenics",+ "canalise": "canalize",+ "canalised": "canalized",+ "canalises": "canalizes",+ "canalising": "canalizing",+ "cancelation": "cancellation",+ "cancelations": "cancellations",+ "cancelled": "canceled",+ "cancelling": "canceling",+ "candour": "candor",+ "cannibalise": "cannibalize",+ "cannibalised": "cannibalized",+ "cannibalises": "cannibalizes",+ "cannibalising": "cannibalizing",+ "canonise": "canonize",+ "canonised": "canonized",+ "canonises": "canonizes",+ "canonising": "canonizing",+ "capitalise": "capitalize",+ "capitalised": "capitalized",+ "capitalises": "capitalizes",+ "capitalising": "capitalizing",+ "caramelise": "caramelize",+ "caramelised": "caramelized",+ "caramelises": "caramelizes",+ "caramelising": "caramelizing",+ "carbonise": "carbonize",+ "carbonised": "carbonized",+ "carbonises": "carbonizes",+ "carbonising": "carbonizing",+ "carolled": "caroled",+ "carolling": "caroling",+ "catalogue": "catalog",+ "catalogued": "cataloged",+ "catalogues": "catalogs",+ "cataloguing": "cataloging",+ "catalyse": "catalyze",+ "catalysed": "catalyzed",+ "catalyses": "catalyzes",+ "catalysing": "catalyzing",+ "categorise": "categorize",+ "categorised": "categorized",+ "categorises": "categorizes",+ "categorising": "categorizing",+ "cauterise": "cauterize",+ "cauterised": "cauterized",+ "cauterises": "cauterizes",+ "cauterising": "cauterizing",+ "cavilled": "caviled",+ "cavilling": "caviling",+ "centigramme": "centigram",+ "centigrammes": "centigrams",+ "centilitre": "centiliter",+ "centilitres": "centiliters",+ "centimetre": "centimeter",+ "centimetres": "centimeters",+ "centralise": "centralize",+ "centralised": "centralized",+ "centralises": "centralizes",+ "centralising": "centralizing",+ "centre": "center",+ "centred": "centered",+ "centrefold": "centerfold",+ "centrefolds": "centerfolds",+ "centrepiece": "centerpiece",+ "centrepieces": "centerpieces",+ "centres": "centers",+ "channelled": "channeled",+ "channelling": "channeling",+ "characterise": "characterize",+ "characterised": "characterized",+ "characterises": "characterizes",+ "characterising": "characterizing",+ "cheque": "check",+ "chequebook": "checkbook",+ "chequebooks": "checkbooks",+ "chequered": "checkered",+ "cheques": "checks",+ "chilli": "chili",+ "chimaera": "chimera",+ "chimaeras": "chimeras",+ "chiselled": "chiseled",+ "chiselling": "chiseling",+ "circularise": "circularize",+ "circularised": "circularized",+ "circularises": "circularizes",+ "circularising": "circularizing",+ "civilise": "civilize",+ "civilised": "civilized",+ "civilises": "civilizes",+ "civilising": "civilizing",+ "clamour": "clamor",+ "clamoured": "clamored",+ "clamouring": "clamoring",+ "clamours": "clamors",+ "clangour": "clangor",+ "clarinettist": "clarinetist",+ "clarinettists": "clarinetists",+ "collectivise": "collectivize",+ "collectivised": "collectivized",+ "collectivises": "collectivizes",+ "collectivising": "collectivizing",+ "colonisation": "colonization",+ "colonise": "colonize",+ "colonised": "colonized",+ "coloniser": "colonizer",+ "colonisers": "colonizers",+ "colonises": "colonizes",+ "colonising": "colonizing",+ "colour": "color",+ "colourant": "colorant",+ "colourants": "colorants",+ "coloured": "colored",+ "coloureds": "coloreds",+ "colourful": "colorful",+ "colourfully": "colorfully",+ "colouring": "coloring",+ "colourize": "colorize",+ "colourized": "colorized",+ "colourizes": "colorizes",+ "colourizing": "colorizing",+ "colourless": "colorless",+ "colours": "colors",+ "commercialise": "commercialize",+ "commercialised": "commercialized",+ "commercialises": "commercializes",+ "commercialising": "commercializing",+ "compartmentalise": "compartmentalize",+ "compartmentalised": "compartmentalized",+ "compartmentalises": "compartmentalizes",+ "compartmentalising": "compartmentalizing",+ "computerise": "computerize",+ "computerised": "computerized",+ "computerises": "computerizes",+ "computerising": "computerizing",+ "conceptualise": "conceptualize",+ "conceptualised": "conceptualized",+ "conceptualises": "conceptualizes",+ "conceptualising": "conceptualizing",+ "connexion": "connection",+ "connexions": "connections",+ "contextualise": "contextualize",+ "contextualised": "contextualized",+ "contextualises": "contextualizes",+ "contextualising": "contextualizing",+ "cosier": "cozier",+ "cosies": "cozies",+ "cosiest": "coziest",+ "cosily": "cozily",+ "cosiness": "coziness",+ "cosy": "cozy",+ "councillor": "councilor",+ "councillors": "councilors",+ "counselled": "counseled",+ "counselling": "counseling",+ "counsellor": "counselor",+ "counsellors": "counselors",+ "crenelated": "crenellated",+ "criminalise": "criminalize",+ "criminalised": "criminalized",+ "criminalises": "criminalizes",+ "criminalising": "criminalizing",+ "criticise": "criticize",+ "criticised": "criticized",+ "criticises": "criticizes",+ "criticising": "criticizing",+ "crueller": "crueler",+ "cruellest": "cruelest",+ "crystallisation": "crystallization",+ "crystallise": "crystallize",+ "crystallised": "crystallized",+ "crystallises": "crystallizes",+ "crystallising": "crystallizing",+ "cudgelled": "cudgeled",+ "cudgelling": "cudgeling",+ "customise": "customize",+ "customised": "customized",+ "customises": "customizes",+ "customising": "customizing",+ "cypher": "cipher",+ "cyphers": "ciphers",+ "decentralisation": "decentralization",+ "decentralise": "decentralize",+ "decentralised": "decentralized",+ "decentralises": "decentralizes",+ "decentralising": "decentralizing",+ "decriminalisation": "decriminalization",+ "decriminalise": "decriminalize",+ "decriminalised": "decriminalized",+ "decriminalises": "decriminalizes",+ "decriminalising": "decriminalizing",+ "defence": "defense",+ "defenceless": "defenseless",+ "defences": "defenses",+ "dehumanisation": "dehumanization",+ "dehumanise": "dehumanize",+ "dehumanised": "dehumanized",+ "dehumanises": "dehumanizes",+ "dehumanising": "dehumanizing",+ "demeanour": "demeanor",+ "demilitarisation": "demilitarization",+ "demilitarise": "demilitarize",+ "demilitarised": "demilitarized",+ "demilitarises": "demilitarizes",+ "demilitarising": "demilitarizing",+ "demobilisation": "demobilization",+ "demobilise": "demobilize",+ "demobilised": "demobilized",+ "demobilises": "demobilizes",+ "demobilising": "demobilizing",+ "democratisation": "democratization",+ "democratise": "democratize",+ "democratised": "democratized",+ "democratises": "democratizes",+ "democratising": "democratizing",+ "demonise": "demonize",+ "demonised": "demonized",+ "demonises": "demonizes",+ "demonising": "demonizing",+ "demoralisation": "demoralization",+ "demoralise": "demoralize",+ "demoralised": "demoralized",+ "demoralises": "demoralizes",+ "demoralising": "demoralizing",+ "denationalisation": "denationalization",+ "denationalise": "denationalize",+ "denationalised": "denationalized",+ "denationalises": "denationalizes",+ "denationalising": "denationalizing",+ "deodorise": "deodorize",+ "deodorised": "deodorized",+ "deodorises": "deodorizes",+ "deodorising": "deodorizing",+ "depersonalise": "depersonalize",+ "depersonalised": "depersonalized",+ "depersonalises": "depersonalizes",+ "depersonalising": "depersonalizing",+ "deputise": "deputize",+ "deputised": "deputized",+ "deputises": "deputizes",+ "deputising": "deputizing",+ "desensitisation": "desensitization",+ "desensitise": "desensitize",+ "desensitised": "desensitized",+ "desensitises": "desensitizes",+ "desensitising": "desensitizing",+ "destabilisation": "destabilization",+ "destabilise": "destabilize",+ "destabilised": "destabilized",+ "destabilises": "destabilizes",+ "destabilising": "destabilizing",+ "dialled": "dialed",+ "dialling": "dialing",+ "dialogue": "dialog",+ "dialogues": "dialogs",+ "diarrhoea": "diarrhea",+ "digitise": "digitize",+ "digitised": "digitized",+ "digitises": "digitizes",+ "digitising": "digitizing",+ "disc": "disk",+ "discolour": "discolor",+ "discoloured": "discolored",+ "discolouring": "discoloring",+ "discolours": "discolors",+ "discs": "disks",+ "disembowelled": "disemboweled",+ "disembowelling": "disemboweling",+ "disfavour": "disfavor",+ "dishevelled": "disheveled",+ "dishonour": "dishonor",+ "dishonourable": "dishonorable",+ "dishonourably": "dishonorably",+ "dishonoured": "dishonored",+ "dishonouring": "dishonoring",+ "dishonours": "dishonors",+ "disorganisation": "disorganization",+ "disorganised": "disorganized",+ "distil": "distill",+ "distils": "distills",+ "dramatisation": "dramatization",+ "dramatisations": "dramatizations",+ "dramatise": "dramatize",+ "dramatised": "dramatized",+ "dramatises": "dramatizes",+ "dramatising": "dramatizing",+ "draught": "draft",+ "draughtboard": "draftboard",+ "draughtboards": "draftboards",+ "draughtier": "draftier",+ "draughtiest": "draftiest",+ "draughts": "drafts",+ "draughtsman": "draftsman",+ "draughtsmanship": "draftsmanship",+ "draughtsmen": "draftsmen",+ "draughtswoman": "draftswoman",+ "draughtswomen": "draftswomen",+ "draughty": "drafty",+ "drivelled": "driveled",+ "drivelling": "driveling",+ "duelled": "dueled",+ "duelling": "dueling",+ "economise": "economize",+ "economised": "economized",+ "economises": "economizes",+ "economising": "economizing",+ "edoema": "edema",+ "editorialise": "editorialize",+ "editorialised": "editorialized",+ "editorialises": "editorializes",+ "editorialising": "editorializing",+ "empathise": "empathize",+ "empathised": "empathized",+ "empathises": "empathizes",+ "empathising": "empathizing",+ "emphasise": "emphasize",+ "emphasised": "emphasized",+ "emphasises": "emphasizes",+ "emphasising": "emphasizing",+ "enamelled": "enameled",+ "enamelling": "enameling",+ "enamoured": "enamored",+ "encyclopaedia": "encyclopedia",+ "encyclopaedias": "encyclopedias",+ "encyclopaedic": "encyclopedic",+ "endeavour": "endeavor",+ "endeavoured": "endeavored",+ "endeavouring": "endeavoring",+ "endeavours": "endeavors",+ "energise": "energize",+ "energised": "energized",+ "energises": "energizes",+ "energising": "energizing",+ "enrol": "enroll",+ "enrols": "enrolls",+ "enthral": "enthrall",+ "enthrals": "enthralls",+ "epaulette": "epaulet",+ "epaulettes": "epaulets",+ "epicentre": "epicenter",+ "epicentres": "epicenters",+ "epilogue": "epilog",+ "epilogues": "epilogs",+ "epitomise": "epitomize",+ "epitomised": "epitomized",+ "epitomises": "epitomizes",+ "epitomising": "epitomizing",+ "equalisation": "equalization",+ "equalise": "equalize",+ "equalised": "equalized",+ "equaliser": "equalizer",+ "equalisers": "equalizers",+ "equalises": "equalizes",+ "equalising": "equalizing",+ "eulogise": "eulogize",+ "eulogised": "eulogized",+ "eulogises": "eulogizes",+ "eulogising": "eulogizing",+ "evangelise": "evangelize",+ "evangelised": "evangelized",+ "evangelises": "evangelizes",+ "evangelising": "evangelizing",+ "exorcise": "exorcize",+ "exorcised": "exorcized",+ "exorcises": "exorcizes",+ "exorcising": "exorcizing",+ "extemporisation": "extemporization",+ "extemporise": "extemporize",+ "extemporised": "extemporized",+ "extemporises": "extemporizes",+ "extemporising": "extemporizing",+ "externalisation": "externalization",+ "externalisations": "externalizations",+ "externalise": "externalize",+ "externalised": "externalized",+ "externalises": "externalizes",+ "externalising": "externalizing",+ "factorise": "factorize",+ "factorised": "factorized",+ "factorises": "factorizes",+ "factorising": "factorizing",+ "faecal": "fecal",+ "faeces": "feces",+ "familiarisation": "familiarization",+ "familiarise": "familiarize",+ "familiarised": "familiarized",+ "familiarises": "familiarizes",+ "familiarising": "familiarizing",+ "fantasise": "fantasize",+ "fantasised": "fantasized",+ "fantasises": "fantasizes",+ "fantasising": "fantasizing",+ "favour": "favor",+ "favourable": "favorable",+ "favourably": "favorably",+ "favoured": "favored",+ "favouring": "favoring",+ "favourite": "favorite",+ "favourites": "favorites",+ "favouritism": "favoritism",+ "favours": "favors",+ "feminise": "feminize",+ "feminised": "feminized",+ "feminises": "feminizes",+ "feminising": "feminizing",+ "fertilisation": "fertilization",+ "fertilise": "fertilize",+ "fertilised": "fertilized",+ "fertiliser": "fertilizer",+ "fertilisers": "fertilizers",+ "fertilises": "fertilizes",+ "fertilising": "fertilizing",+ "fervour": "fervor",+ "fibre": "fiber",+ "fibreglass": "fiberglass",+ "fibres": "fibers",+ "fictionalisation": "fictionalization",+ "fictionalisations": "fictionalizations",+ "fictionalise": "fictionalize",+ "fictionalised": "fictionalized",+ "fictionalises": "fictionalizes",+ "fictionalising": "fictionalizing",+ "fillet": "filet",+ "filleted": "fileted",+ "filleting": "fileting",+ "fillets": "filets",+ "finalisation": "finalization",+ "finalise": "finalize",+ "finalised": "finalized",+ "finalises": "finalizes",+ "finalising": "finalizing",+ "flautist": "flutist",+ "flautists": "flutists",+ "flavour": "flavor",+ "flavoured": "flavored",+ "flavouring": "flavoring",+ "flavourings": "flavorings",+ "flavourless": "flavorless",+ "flavours": "flavors",+ "flavoursome": "flavorsome",+ "flyer / flier": "flier / flyer",+ "foetal": "fetal",+ "foetid": "fetid",+ "foetus": "fetus",+ "foetuses": "fetuses",+ "formalisation": "formalization",+ "formalise": "formalize",+ "formalised": "formalized",+ "formalises": "formalizes",+ "formalising": "formalizing",+ "fossilisation": "fossilization",+ "fossilise": "fossilize",+ "fossilised": "fossilized",+ "fossilises": "fossilizes",+ "fossilising": "fossilizing",+ "fraternisation": "fraternization",+ "fraternise": "fraternize",+ "fraternised": "fraternized",+ "fraternises": "fraternizes",+ "fraternising": "fraternizing",+ "fulfil": "fulfill",+ "fulfilment": "fulfillment",+ "fulfils": "fulfills",+ "funnelled": "funneled",+ "funnelling": "funneling",+ "galvanise": "galvanize",+ "galvanised": "galvanized",+ "galvanises": "galvanizes",+ "galvanising": "galvanizing",+ "gambolled": "gamboled",+ "gambolling": "gamboling",+ "gaol": "jail",+ "gaolbird": "jailbird",+ "gaolbirds": "jailbirds",+ "gaolbreak": "jailbreak",+ "gaolbreaks": "jailbreaks",+ "gaoled": "jailed",+ "gaoler": "jailer",+ "gaolers": "jailers",+ "gaoling": "jailing",+ "gaols": "jails",+ "gasses": "gases",+ "gage": "gauge",+ "gaged": "gauged",+ "gages": "gauges",+ "gaging": "gauging",+ "generalisation": "generalization",+ "generalisations": "generalizations",+ "generalise": "generalize",+ "generalised": "generalized",+ "generalises": "generalizes",+ "generalising": "generalizing",+ "ghettoise": "ghettoize",+ "ghettoised": "ghettoized",+ "ghettoises": "ghettoizes",+ "ghettoising": "ghettoizing",+ "gipsies": "gypsies",+ "glamorise": "glamorize",+ "glamorised": "glamorized",+ "glamorises": "glamorizes",+ "glamorising": "glamorizing",+ "glamor": "glamour",+ "globalisation": "globalization",+ "globalise": "globalize",+ "globalised": "globalized",+ "globalises": "globalizes",+ "globalising": "globalizing",+ "glueing": "gluing",+ "goitre": "goiter",+ "goitres": "goiters",+ "gonorrhoea": "gonorrhea",+ "gramme": "gram",+ "grammes": "grams",+ "gravelled": "graveled",+ "grey": "gray",+ "greyed": "grayed",+ "greying": "graying",+ "greyish": "grayish",+ "greyness": "grayness",+ "greys": "grays",+ "grovelled": "groveled",+ "grovelling": "groveling",+ "groyne": "groin",+ "groynes": "groins",+ "gruelling": "grueling",+ "gruellingly": "gruelingly",+ "gryphon": "griffin",+ "gryphons": "griffins",+ "gynaecological": "gynecological",+ "gynaecologist": "gynecologist",+ "gynaecologists": "gynecologists",+ "gynaecology": "gynecology",+ "haematological": "hematological",+ "haematologist": "hematologist",+ "haematologists": "hematologists",+ "haematology": "hematology",+ "haemoglobin": "hemoglobin",+ "haemophilia": "hemophilia",+ "haemophiliac": "hemophiliac",+ "haemophiliacs": "hemophiliacs",+ "haemorrhage": "hemorrhage",+ "haemorrhaged": "hemorrhaged",+ "haemorrhages": "hemorrhages",+ "haemorrhaging": "hemorrhaging",+ "haemorrhoids": "hemorrhoids",+ "harbour": "harbor",+ "harboured": "harbored",+ "harbouring": "harboring",+ "harbours": "harbors",+ "harmonisation": "harmonization",+ "harmonise": "harmonize",+ "harmonised": "harmonized",+ "harmonises": "harmonizes",+ "harmonising": "harmonizing",+ "homoeopath": "homeopath",+ "homoeopathic": "homeopathic",+ "homoeopaths": "homeopaths",+ "homoeopathy": "homeopathy",+ "homogenise": "homogenize",+ "homogenised": "homogenized",+ "homogenises": "homogenizes",+ "homogenising": "homogenizing",+ "honour": "honor",+ "honourable": "honorable",+ "honourably": "honorably",+ "honoured": "honored",+ "honouring": "honoring",+ "honours": "honors",+ "hospitalisation": "hospitalization",+ "hospitalise": "hospitalize",+ "hospitalised": "hospitalized",+ "hospitalises": "hospitalizes",+ "hospitalising": "hospitalizing",+ "humanise": "humanize",+ "humanised": "humanized",+ "humanises": "humanizes",+ "humanising": "humanizing",+ "humour": "humor",+ "humoured": "humored",+ "humouring": "humoring",+ "humourless": "humorless",+ "humours": "humors",+ "hybridise": "hybridize",+ "hybridised": "hybridized",+ "hybridises": "hybridizes",+ "hybridising": "hybridizing",+ "hypnotise": "hypnotize",+ "hypnotised": "hypnotized",+ "hypnotises": "hypnotizes",+ "hypnotising": "hypnotizing",+ "hypothesise": "hypothesize",+ "hypothesised": "hypothesized",+ "hypothesises": "hypothesizes",+ "hypothesising": "hypothesizing",+ "idealisation": "idealization",+ "idealise": "idealize",+ "idealised": "idealized",+ "idealises": "idealizes",+ "idealising": "idealizing",+ "idolise": "idolize",+ "idolised": "idolized",+ "idolises": "idolizes",+ "idolising": "idolizing",+ "immobilisation": "immobilization",+ "immobilise": "immobilize",+ "immobilised": "immobilized",+ "immobiliser": "immobilizer",+ "immobilisers": "immobilizers",+ "immobilises": "immobilizes",+ "immobilising": "immobilizing",+ "immortalise": "immortalize",+ "immortalised": "immortalized",+ "immortalises": "immortalizes",+ "immortalising": "immortalizing",+ "immunisation": "immunization",+ "immunise": "immunize",+ "immunised": "immunized",+ "immunises": "immunizes",+ "immunising": "immunizing",+ "impanelled": "impaneled",+ "impanelling": "impaneling",+ "imperilled": "imperiled",+ "imperilling": "imperiling",+ "individualise": "individualize",+ "individualised": "individualized",+ "individualises": "individualizes",+ "individualising": "individualizing",+ "industrialise": "industrialize",+ "industrialised": "industrialized",+ "industrialises": "industrializes",+ "industrialising": "industrializing",+ "inflexion": "inflection",+ "inflexions": "inflections",+ "initialise": "initialize",+ "initialised": "initialized",+ "initialises": "initializes",+ "initialising": "initializing",+ "initialled": "initialed",+ "initialling": "initialing",+ "instal": "install",+ "instalment": "installment",+ "instalments": "installments",+ "instals": "installs",+ "instil": "instill",+ "instils": "instills",+ "institutionalisation": "institutionalization",+ "institutionalise": "institutionalize",+ "institutionalised": "institutionalized",+ "institutionalises": "institutionalizes",+ "institutionalising": "institutionalizing",+ "intellectualise": "intellectualize",+ "intellectualised": "intellectualized",+ "intellectualises": "intellectualizes",+ "intellectualising": "intellectualizing",+ "internalisation": "internalization",+ "internalise": "internalize",+ "internalised": "internalized",+ "internalises": "internalizes",+ "internalising": "internalizing",+ "internationalisation": "internationalization",+ "internationalise": "internationalize",+ "internationalised": "internationalized",+ "internationalises": "internationalizes",+ "internationalising": "internationalizing",+ "ionisation": "ionization",+ "ionise": "ionize",+ "ionised": "ionized",+ "ioniser": "ionizer",+ "ionisers": "ionizers",+ "ionises": "ionizes",+ "ionising": "ionizing",+ "italicise": "italicize",+ "italicised": "italicized",+ "italicises": "italicizes",+ "italicising": "italicizing",+ "itemise": "itemize",+ "itemised": "itemized",+ "itemises": "itemizes",+ "itemising": "itemizing",+ "jeopardise": "jeopardize",+ "jeopardised": "jeopardized",+ "jeopardises": "jeopardizes",+ "jeopardising": "jeopardizing",+ "jewelled": "jeweled",+ "jeweller": "jeweler",+ "jewellers": "jewelers",+ "jewellery": "jewelry",+ "judgement": "judgment",+ "kilogramme": "kilogram",+ "kilogrammes": "kilograms",+ "kilometre": "kilometer",+ "kilometres": "kilometers",+ "labelled": "labeled",+ "labelling": "labeling",+ "labour": "labor",+ "laboured": "labored",+ "labourer": "laborer",+ "labourers": "laborers",+ "labouring": "laboring",+ "labours": "labors",+ "lacklustre": "lackluster",+ "legalisation": "legalization",+ "legalise": "legalize",+ "legalised": "legalized",+ "legalises": "legalizes",+ "legalising": "legalizing",+ "legitimise": "legitimize",+ "legitimised": "legitimized",+ "legitimises": "legitimizes",+ "legitimising": "legitimizing",+ "leukaemia": "leukemia",+ "levelled": "leveled",+ "leveller": "leveler",+ "levellers": "levelers",+ "levelling": "leveling",+ "libelled": "libeled",+ "libelling": "libeling",+ "libellous": "libelous",+ "liberalisation": "liberalization",+ "liberalise": "liberalize",+ "liberalised": "liberalized",+ "liberalises": "liberalizes",+ "liberalising": "liberalizing",+ "licence": "license",+ "licenced": "licensed",+ "licences": "licenses",+ "licencing": "licensing",+ "likeable": "likable",+ "lionisation": "lionization",+ "lionise": "lionize",+ "lionised": "lionized",+ "lionises": "lionizes",+ "lionising": "lionizing",+ "liquidise": "liquidize",+ "liquidised": "liquidized",+ "liquidiser": "liquidizer",+ "liquidisers": "liquidizers",+ "liquidises": "liquidizes",+ "liquidising": "liquidizing",+ "litre": "liter",+ "litres": "liters",+ "localise": "localize",+ "localised": "localized",+ "localises": "localizes",+ "localising": "localizing",+ "louvre": "louver",+ "louvred": "louvered",+ "louvres": "louvers",+ "lustre": "luster",+ "magnetise": "magnetize",+ "magnetised": "magnetized",+ "magnetises": "magnetizes",+ "magnetising": "magnetizing",+ "manoeuvrability": "maneuverability",+ "manoeuvrable": "maneuverable",+ "manoeuvre": "maneuver",+ "manoeuvred": "maneuvered",+ "manoeuvres": "maneuvers",+ "manoeuvring": "maneuvering",+ "manoeuvrings": "maneuverings",+ "marginalisation": "marginalization",+ "marginalise": "marginalize",+ "marginalised": "marginalized",+ "marginalises": "marginalizes",+ "marginalising": "marginalizing",+ "marshalled": "marshaled",+ "marshalling": "marshaling",+ "marvelled": "marveled",+ "marvelling": "marveling",+ "marvellous": "marvelous",+ "marvellously": "marvelously",+ "materialisation": "materialization",+ "materialise": "materialize",+ "materialised": "materialized",+ "materialises": "materializes",+ "materialising": "materializing",+ "maximisation": "maximization",+ "maximise": "maximize",+ "maximised": "maximized",+ "maximises": "maximizes",+ "maximising": "maximizing",+ "meagre": "meager",+ "mechanisation": "mechanization",+ "mechanise": "mechanize",+ "mechanised": "mechanized",+ "mechanises": "mechanizes",+ "mechanising": "mechanizing",+ "mediaeval": "medieval",+ "memorialise": "memorialize",+ "memorialised": "memorialized",+ "memorialises": "memorializes",+ "memorialising": "memorializing",+ "memorise": "memorize",+ "memorised": "memorized",+ "memorises": "memorizes",+ "memorising": "memorizing",+ "mesmerise": "mesmerize",+ "mesmerised": "mesmerized",+ "mesmerises": "mesmerizes",+ "mesmerising": "mesmerizing",+ "metabolise": "metabolize",+ "metabolised": "metabolized",+ "metabolises": "metabolizes",+ "metabolising": "metabolizing",+ "metre": "meter",+ "metres": "meters",+ "micrometre": "micrometer",+ "micrometres": "micrometers",+ "militarise": "militarize",+ "militarised": "militarized",+ "militarises": "militarizes",+ "militarising": "militarizing",+ "milligramme": "milligram",+ "milligrammes": "milligrams",+ "millilitre": "milliliter",+ "millilitres": "milliliters",+ "millimetre": "millimeter",+ "millimetres": "millimeters",+ "miniaturisation": "miniaturization",+ "miniaturise": "miniaturize",+ "miniaturised": "miniaturized",+ "miniaturises": "miniaturizes",+ "miniaturising": "miniaturizing",+ "minibusses": "minibuses",+ "minimise": "minimize",+ "minimised": "minimized",+ "minimises": "minimizes",+ "minimising": "minimizing",+ "misbehaviour": "misbehavior",+ "misdemeanour": "misdemeanor",+ "misdemeanours": "misdemeanors",+ "misspelt": "misspelled",+ "mitre": "miter",+ "mitres": "miters",+ "mobilisation": "mobilization",+ "mobilise": "mobilize",+ "mobilised": "mobilized",+ "mobilises": "mobilizes",+ "mobilising": "mobilizing",+ "modelled": "modeled",+ "modeller": "modeler",+ "modellers": "modelers",+ "modelling": "modeling",+ "modernise": "modernize",+ "modernised": "modernized",+ "modernises": "modernizes",+ "modernising": "modernizing",+ "moisturise": "moisturize",+ "moisturised": "moisturized",+ "moisturiser": "moisturizer",+ "moisturisers": "moisturizers",+ "moisturises": "moisturizes",+ "moisturising": "moisturizing",+ "monologue": "monolog",+ "monologues": "monologs",+ "monopolisation": "monopolization",+ "monopolise": "monopolize",+ "monopolised": "monopolized",+ "monopolises": "monopolizes",+ "monopolising": "monopolizing",+ "moralise": "moralize",+ "moralised": "moralized",+ "moralises": "moralizes",+ "moralising": "moralizing",+ "motorised": "motorized",+ "mould": "mold",+ "moulded": "molded",+ "moulder": "molder",+ "mouldered": "moldered",+ "mouldering": "moldering",+ "moulders": "molders",+ "mouldier": "moldier",+ "mouldiest": "moldiest",+ "moulding": "molding",+ "mouldings": "moldings",+ "moulds": "molds",+ "mouldy": "moldy",+ "moult": "molt",+ "moulted": "molted",+ "moulting": "molting",+ "moults": "molts",+ "moustache": "mustache",+ "moustached": "mustached",+ "moustaches": "mustaches",+ "moustachioed": "mustachioed",+ "multicoloured": "multicolored",+ "nationalisation": "nationalization",+ "nationalisations": "nationalizations",+ "nationalise": "nationalize",+ "nationalised": "nationalized",+ "nationalises": "nationalizes",+ "nationalising": "nationalizing",+ "naturalisation": "naturalization",+ "naturalise": "naturalize",+ "naturalised": "naturalized",+ "naturalises": "naturalizes",+ "naturalising": "naturalizing",+ "neighbour": "neighbor",+ "neighbourhood": "neighborhood",+ "neighbourhoods": "neighborhoods",+ "neighbouring": "neighboring",+ "neighbourliness": "neighborliness",+ "neighbourly": "neighborly",+ "neighbours": "neighbors",+ "neutralisation": "neutralization",+ "neutralise": "neutralize",+ "neutralised": "neutralized",+ "neutralises": "neutralizes",+ "neutralising": "neutralizing",+ "normalisation": "normalization",+ "normalise": "normalize",+ "normalised": "normalized",+ "normalises": "normalizes",+ "normalising": "normalizing",+ "odour": "odor",+ "odourless": "odorless",+ "odours": "odors",+ "oesophagus": "esophagus",+ "oesophaguses": "esophaguses",+ "oestrogen": "estrogen",+ "offence": "offense",+ "offences": "offenses",+ "omelette": "omelet",+ "omelettes": "omelets",+ "optimise": "optimize",+ "optimised": "optimized",+ "optimises": "optimizes",+ "optimising": "optimizing",+ "organisation": "organization",+ "organisational": "organizational",+ "organisations": "organizations",+ "organise": "organize",+ "organised": "organized",+ "organiser": "organizer",+ "organisers": "organizers",+ "organises": "organizes",+ "organising": "organizing",+ "orthopaedic": "orthopedic",+ "orthopaedics": "orthopedics",+ "ostracise": "ostracize",+ "ostracised": "ostracized",+ "ostracises": "ostracizes",+ "ostracising": "ostracizing",+ "outmanoeuvre": "outmaneuver",+ "outmanoeuvred": "outmaneuvered",+ "outmanoeuvres": "outmaneuvers",+ "outmanoeuvring": "outmaneuvering",+ "overemphasise": "overemphasize",+ "overemphasised": "overemphasized",+ "overemphasises": "overemphasizes",+ "overemphasising": "overemphasizing",+ "oxidisation": "oxidization",+ "oxidise": "oxidize",+ "oxidised": "oxidized",+ "oxidises": "oxidizes",+ "oxidising": "oxidizing",+ "paederast": "pederast",+ "paederasts": "pederasts",+ "paediatric": "pediatric",+ "paediatrician": "pediatrician",+ "paediatricians": "pediatricians",+ "paediatrics": "pediatrics",+ "paedophile": "pedophile",+ "paedophiles": "pedophiles",+ "paedophilia": "pedophilia",+ "palaeolithic": "paleolithic",+ "palaeontologist": "paleontologist",+ "palaeontologists": "paleontologists",+ "palaeontology": "paleontology",+ "panelled": "paneled",+ "panelling": "paneling",+ "panellist": "panelist",+ "panellists": "panelists",+ "paralyse": "paralyze",+ "paralysed": "paralyzed",+ "paralyses": "paralyzes",+ "paralysing": "paralyzing",+ "parcelled": "parceled",+ "parcelling": "parceling",+ "parlour": "parlor",+ "parlours": "parlors",+ "particularise": "particularize",+ "particularised": "particularized",+ "particularises": "particularizes",+ "particularising": "particularizing",+ "passivisation": "passivization",+ "passivise": "passivize",+ "passivised": "passivized",+ "passivises": "passivizes",+ "passivising": "passivizing",+ "pasteurisation": "pasteurization",+ "pasteurise": "pasteurize",+ "pasteurised": "pasteurized",+ "pasteurises": "pasteurizes",+ "pasteurising": "pasteurizing",+ "patronise": "patronize",+ "patronised": "patronized",+ "patronises": "patronizes",+ "patronising": "patronizing",+ "patronisingly": "patronizingly",+ "pedalled": "pedaled",+ "pedalling": "pedaling",+ "pedestrianisation": "pedestrianization",+ "pedestrianise": "pedestrianize",+ "pedestrianised": "pedestrianized",+ "pedestrianises": "pedestrianizes",+ "pedestrianising": "pedestrianizing",+ "penalise": "penalize",+ "penalised": "penalized",+ "penalises": "penalizes",+ "penalising": "penalizing",+ "pencilled": "penciled",+ "pencilling": "penciling",+ "personalise": "personalize",+ "personalised": "personalized",+ "personalises": "personalizes",+ "personalising": "personalizing",+ "pharmacopoeia": "pharmacopeia",+ "pharmacopoeias": "pharmacopeias",+ "philosophise": "philosophize",+ "philosophised": "philosophized",+ "philosophises": "philosophizes",+ "philosophising": "philosophizing",+ "philtre": "filter",+ "philtres": "filters",+ "phoney": "phony",+ "plagiarise": "plagiarize",+ "plagiarised": "plagiarized",+ "plagiarises": "plagiarizes",+ "plagiarising": "plagiarizing",+ "plough": "plow",+ "ploughed": "plowed",+ "ploughing": "plowing",+ "ploughman": "plowman",+ "ploughmen": "plowmen",+ "ploughs": "plows",+ "ploughshare": "plowshare",+ "ploughshares": "plowshares",+ "polarisation": "polarization",+ "polarise": "polarize",+ "polarised": "polarized",+ "polarises": "polarizes",+ "polarising": "polarizing",+ "politicisation": "politicization",+ "politicise": "politicize",+ "politicised": "politicized",+ "politicises": "politicizes",+ "politicising": "politicizing",+ "popularisation": "popularization",+ "popularise": "popularize",+ "popularised": "popularized",+ "popularises": "popularizes",+ "popularising": "popularizing",+ "pouffe": "pouf",+ "pouffes": "poufs",+ "practise": "practice",+ "practised": "practiced",+ "practises": "practices",+ "practising": "practicing",+ "praesidium": "presidium",+ "praesidiums": "presidiums",+ "pressurisation": "pressurization",+ "pressurise": "pressurize",+ "pressurised": "pressurized",+ "pressurises": "pressurizes",+ "pressurising": "pressurizing",+ "pretence": "pretense",+ "pretences": "pretenses",+ "primaeval": "primeval",+ "prioritisation": "prioritization",+ "prioritise": "prioritize",+ "prioritised": "prioritized",+ "prioritises": "prioritizes",+ "prioritising": "prioritizing",+ "privatisation": "privatization",+ "privatisations": "privatizations",+ "privatise": "privatize",+ "privatised": "privatized",+ "privatises": "privatizes",+ "privatising": "privatizing",+ "professionalisation": "professionalization",+ "professionalise": "professionalize",+ "professionalised": "professionalized",+ "professionalises": "professionalizes",+ "professionalising": "professionalizing",+ "programme": "program",+ "programmes": "programs",+ "prologue": "prolog",+ "prologues": "prologs",+ "propagandise": "propagandize",+ "propagandised": "propagandized",+ "propagandises": "propagandizes",+ "propagandising": "propagandizing",+ "proselytise": "proselytize",+ "proselytised": "proselytized",+ "proselytiser": "proselytizer",+ "proselytisers": "proselytizers",+ "proselytises": "proselytizes",+ "proselytising": "proselytizing",+ "psychoanalyse": "psychoanalyze",+ "psychoanalysed": "psychoanalyzed",+ "psychoanalyses": "psychoanalyzes",+ "psychoanalysing": "psychoanalyzing",+ "publicise": "publicize",+ "publicised": "publicized",+ "publicises": "publicizes",+ "publicising": "publicizing",+ "pulverisation": "pulverization",+ "pulverise": "pulverize",+ "pulverised": "pulverized",+ "pulverises": "pulverizes",+ "pulverising": "pulverizing",+ "pummelled": "pummel",+ "pummelling": "pummeled",+ "pyjama": "pajama",+ "pyjamas": "pajamas",+ "pzazz": "pizzazz",+ "quarrelled": "quarreled",+ "quarrelling": "quarreling",+ "radicalise": "radicalize",+ "radicalised": "radicalized",+ "radicalises": "radicalizes",+ "radicalising": "radicalizing",+ "rancour": "rancor",+ "randomise": "randomize",+ "randomised": "randomized",+ "randomises": "randomizes",+ "randomising": "randomizing",+ "rationalisation": "rationalization",+ "rationalisations": "rationalizations",+ "rationalise": "rationalize",+ "rationalised": "rationalized",+ "rationalises": "rationalizes",+ "rationalising": "rationalizing",+ "ravelled": "raveled",+ "ravelling": "raveling",+ "realisable": "realizable",+ "realisation": "realization",+ "realisations": "realizations",+ "realise": "realize",+ "realised": "realized",+ "realises": "realizes",+ "realising": "realizing",+ "recognisable": "recognizable",+ "recognisably": "recognizably",+ "recognisance": "recognizance",+ "recognise": "recognize",+ "recognised": "recognized",+ "recognises": "recognizes",+ "recognising": "recognizing",+ "reconnoitre": "reconnoiter",+ "reconnoitred": "reconnoitered",+ "reconnoitres": "reconnoiters",+ "reconnoitring": "reconnoitering",+ "refuelled": "refueled",+ "refuelling": "refueling",+ "regularisation": "regularization",+ "regularise": "regularize",+ "regularised": "regularized",+ "regularises": "regularizes",+ "regularising": "regularizing",+ "remodelled": "remodeled",+ "remodelling": "remodeling",+ "remould": "remold",+ "remoulded": "remolded",+ "remoulding": "remolding",+ "remoulds": "remolds",+ "reorganisation": "reorganization",+ "reorganisations": "reorganizations",+ "reorganise": "reorganize",+ "reorganised": "reorganized",+ "reorganises": "reorganizes",+ "reorganising": "reorganizing",+ "revelled": "reveled",+ "reveller": "reveler",+ "revellers": "revelers",+ "revelling": "reveling",+ "revitalise": "revitalize",+ "revitalised": "revitalized",+ "revitalises": "revitalizes",+ "revitalising": "revitalizing",+ "revolutionise": "revolutionize",+ "revolutionised": "revolutionized",+ "revolutionises": "revolutionizes",+ "revolutionising": "revolutionizing",+ "rhapsodise": "rhapsodize",+ "rhapsodised": "rhapsodized",+ "rhapsodises": "rhapsodizes",+ "rhapsodising": "rhapsodizing",+ "rigour": "rigor",+ "rigours": "rigors",+ "ritualised": "ritualized",+ "rivalled": "rivaled",+ "rivalling": "rivaling",+ "romanticise": "romanticize",+ "romanticised": "romanticized",+ "romanticises": "romanticizes",+ "romanticising": "romanticizing",+ "rumour": "rumor",+ "rumoured": "rumored",+ "rumours": "rumors",+ "sabre": "saber",+ "sabres": "sabers",+ "saltpetre": "saltpeter",+ "sanitise": "sanitize",+ "sanitised": "sanitized",+ "sanitises": "sanitizes",+ "sanitising": "sanitizing",+ "satirise": "satirize",+ "satirised": "satirized",+ "satirises": "satirizes",+ "satirising": "satirizing",+ "saviour": "savior",+ "saviours": "saviors",+ "savour": "savor",+ "savoured": "savored",+ "savouries": "savories",+ "savouring": "savoring",+ "savours": "savors",+ "savoury": "savory",+ "scandalise": "scandalize",+ "scandalised": "scandalized",+ "scandalises": "scandalizes",+ "scandalising": "scandalizing",+ "sceptic": "skeptic",+ "sceptical": "skeptical",+ "sceptically": "skeptically",+ "scepticism": "skepticism",+ "sceptics": "skeptics",+ "sceptre": "scepter",+ "sceptres": "scepters",+ "scrutinise": "scrutinize",+ "scrutinised": "scrutinized",+ "scrutinises": "scrutinizes",+ "scrutinising": "scrutinizing",+ "secularisation": "secularization",+ "secularise": "secularize",+ "secularised": "secularized",+ "secularises": "secularizes",+ "secularising": "secularizing",+ "sensationalise": "sensationalize",+ "sensationalised": "sensationalized",+ "sensationalises": "sensationalizes",+ "sensationalising": "sensationalizing",+ "sensitise": "sensitize",+ "sensitised": "sensitized",+ "sensitises": "sensitizes",+ "sensitising": "sensitizing",+ "sentimentalise": "sentimentalize",+ "sentimentalised": "sentimentalized",+ "sentimentalises": "sentimentalizes",+ "sentimentalising": "sentimentalizing",+ "sepulchre": "sepulcher",+ "sepulchres": "sepulchers",+ "serialisation": "serialization",+ "serialisations": "serializations",+ "serialise": "serialize",+ "serialised": "serialized",+ "serialises": "serializes",+ "serialising": "serializing",+ "sermonise": "sermonize",+ "sermonised": "sermonized",+ "sermonises": "sermonizes",+ "sermonising": "sermonizing",+ "sheikh": "sheik",+ "shovelled": "shoveled",+ "shovelling": "shoveling",+ "shrivelled": "shriveled",+ "shrivelling": "shriveling",+ "signalise": "signalize",+ "signalised": "signalized",+ "signalises": "signalizes",+ "signalising": "signalizing",+ "signalled": "signaled",+ "signalling": "signaling",+ "smoulder": "smolder",+ "smouldered": "smoldered",+ "smouldering": "smoldering",+ "smoulders": "smolders",+ "snivelled": "sniveled",+ "snivelling": "sniveling",+ "snorkelled": "snorkeled",+ "snorkelling": "snorkeling",+ "snowplough": "snowplow",+ "snowploughs": "snowplow",+ "socialisation": "socialization",+ "socialise": "socialize",+ "socialised": "socialized",+ "socialises": "socializes",+ "socialising": "socializing",+ "sodomise": "sodomize",+ "sodomised": "sodomized",+ "sodomises": "sodomizes",+ "sodomising": "sodomizing",+ "solemnise": "solemnize",+ "solemnised": "solemnized",+ "solemnises": "solemnizes",+ "solemnising": "solemnizing",+ "sombre": "somber",+ "specialisation": "specialization",+ "specialisations": "specializations",+ "specialise": "specialize",+ "specialised": "specialized",+ "specialises": "specializes",+ "specialising": "specializing",+ "spectre": "specter",+ "spectres": "specters",+ "spiralled": "spiraled",+ "spiralling": "spiraling",+ "splendour": "splendor",+ "splendours": "splendors",+ "squirrelled": "squirreled",+ "squirrelling": "squirreling",+ "stabilisation": "stabilization",+ "stabilise": "stabilize",+ "stabilised": "stabilized",+ "stabiliser": "stabilizer",+ "stabilisers": "stabilizers",+ "stabilises": "stabilizes",+ "stabilising": "stabilizing",+ "standardisation": "standardization",+ "standardise": "standardize",+ "standardised": "standardized",+ "standardises": "standardizes",+ "standardising": "standardizing",+ "stencilled": "stenciled",+ "stencilling": "stenciling",+ "sterilisation": "sterilization",+ "sterilisations": "sterilizations",+ "sterilise": "sterilize",+ "sterilised": "sterilized",+ "steriliser": "sterilizer",+ "sterilisers": "sterilizers",+ "sterilises": "sterilizes",+ "sterilising": "sterilizing",+ "stigmatisation": "stigmatization",+ "stigmatise": "stigmatize",+ "stigmatised": "stigmatized",+ "stigmatises": "stigmatizes",+ "stigmatising": "stigmatizing",+ "storey": "story",+ "storeys": "stories",+ "subsidisation": "subsidization",+ "subsidise": "subsidize",+ "subsidised": "subsidized",+ "subsidiser": "subsidizer",+ "subsidisers": "subsidizers",+ "subsidises": "subsidizes",+ "subsidising": "subsidizing",+ "succour": "succor",+ "succoured": "succored",+ "succouring": "succoring",+ "succours": "succors",+ "sulphate": "sulfate",+ "sulphates": "sulfates",+ "sulphide": "sulfide",+ "sulphides": "sulfides",+ "sulphur": "sulfur",+ "sulphurous": "sulfurous",+ "summarise": "summarize",+ "summarised": "summarized",+ "summarises": "summarizes",+ "summarising": "summarizing",+ "swivelled": "swiveled",+ "swivelling": "swiveling",+ "symbolise": "symbolize",+ "symbolised": "symbolized",+ "symbolises": "symbolizes",+ "symbolising": "symbolizing",+ "sympathise": "sympathize",+ "sympathised": "sympathized",+ "sympathiser": "sympathizer",+ "sympathisers": "sympathizers",+ "sympathises": "sympathizes",+ "sympathising": "sympathizing",+ "synchronisation": "synchronization",+ "synchronise": "synchronize",+ "synchronised": "synchronized",+ "synchronises": "synchronizes",+ "synchronising": "synchronizing",+ "synthesise": "synthesize",+ "synthesised": "synthesized",+ "synthesiser": "synthesizer",+ "synthesisers": "synthesizers",+ "synthesises": "synthesizes",+ "synthesising": "synthesizing",+ "syphon": "siphon",+ "syphoned": "siphoned",+ "syphoning": "siphoning",+ "syphons": "siphons",+ "systematisation": "systematization",+ "systematise": "systematize",+ "systematised": "systematized",+ "systematises": "systematizes",+ "systematising": "systematizing",+ "tantalise": "tantalize",+ "tantalised": "tantalized",+ "tantalises": "tantalizes",+ "tantalising": "tantalizing",+ "tantalisingly": "tantalizingly",+ "tasselled": "tasseled",+ "technicolour": "technicolor",+ "temporise": "temporize",+ "temporised": "temporized",+ "temporises": "temporizes",+ "temporising": "temporizing",+ "tenderise": "tenderize",+ "tenderised": "tenderized",+ "tenderises": "tenderizes",+ "tenderising": "tenderizing",+ "terrorise": "terrorize",+ "terrorised": "terrorized",+ "terrorises": "terrorizes",+ "terrorising": "terrorizing",+ "theatre": "theater",+ "theatregoer": "theatergoer",+ "theatregoers": "theatergoers",+ "theatres": "theaters",+ "theorise": "theorize",+ "theorised": "theorized",+ "theorises": "theorizes",+ "theorising": "theorizing",+ "tonne": "ton",+ "tonnes": "tons",+ "towelled": "toweled",+ "towelling": "toweling",+ "toxaemia": "toxemia",+ "tranquillise": "tranquilize",+ "tranquillised": "tranquilized",+ "tranquilliser": "tranquilizer",+ "tranquillisers": "tranquilizers",+ "tranquillises": "tranquilizes",+ "tranquillising": "tranquilizing",+ "tranquillity": "tranquility",+ "tranquillize": "tranquilize",+ "tranquillized": "tranquilized",+ "tranquillizer": "tranquilizer",+ "tranquillizers": "tranquilizers",+ "tranquillizes": "tranquilizes",+ "tranquillizing": "tranquilizing",+ "tranquilly": "tranquility",+ "transistorised": "transistorized",+ "traumatise": "traumatize",+ "traumatised": "traumatized",+ "traumatises": "traumatizes",+ "traumatising": "traumatizing",+ "travelled": "traveled",+ "traveller": "traveler",+ "travellers": "travelers",+ "travelling": "traveling",+ "travelog": "travelogue",+ "travelogs": "travelogues",+ "trialled": "trialed",+ "trialling": "trialing",+ "tricolour": "tricolor",+ "tricolours": "tricolors",+ "trivialise": "trivialize",+ "trivialised": "trivialized",+ "trivialises": "trivializes",+ "trivialising": "trivializing",+ "tumour": "tumor",+ "tumours": "tumors",+ "tunnelled": "tunneled",+ "tunnelling": "tunneling",+ "tyrannise": "tyrannize",+ "tyrannised": "tyrannized",+ "tyrannises": "tyrannizes",+ "tyrannising": "tyrannizing",+ "tyre": "tire",+ "tyres": "tires",+ "unauthorised": "unauthorized",+ "uncivilised": "uncivilized",+ "underutilised": "underutilized",+ "unequalled": "unequaled",+ "unfavourable": "unfavorable",+ "unfavourably": "unfavorably",+ "unionisation": "unionization",+ "unionise": "unionize",+ "unionised": "unionized",+ "unionises": "unionizes",+ "unionising": "unionizing",+ "unorganised": "unorganized",+ "unravelled": "unraveled",+ "unravelling": "unraveling",+ "unrecognisable": "unrecognizable",+ "unrecognised": "unrecognized",+ "unrivalled": "unrivaled",+ "unsavoury": "unsavory",+ "untrammelled": "untrammeled",+ "urbanisation": "urbanization",+ "urbanise": "urbanize",+ "urbanised": "urbanized",+ "urbanises": "urbanizes",+ "urbanising": "urbanizing",+ "utilisable": "utilizable",+ "utilisation": "utilization",+ "utilise": "utilize",+ "utilised": "utilized",+ "utilises": "utilizes",+ "utilising": "utilizing",+ "valour": "valor",+ "vandalise": "vandalize",+ "vandalised": "vandalized",+ "vandalises": "vandalizes",+ "vandalising": "vandalizing",+ "vaporisation": "vaporization",+ "vaporise": "vaporize",+ "vaporised": "vaporized",+ "vaporises": "vaporizes",+ "vaporising": "vaporizing",+ "vapour": "vapor",+ "vapours": "vapors",+ "verbalise": "verbalize",+ "verbalised": "verbalized",+ "verbalises": "verbalizes",+ "verbalising": "verbalizing",+ "victimisation": "victimization",+ "victimise": "victimize",+ "victimised": "victimized",+ "victimises": "victimizes",+ "victimising": "victimizing",+ "videodisc": "videodisk",+ "videodiscs": "videodisks",+ "vigour": "vigor",+ "visualisation": "visualization",+ "visualisations": "visualizations",+ "visualise": "visualize",+ "visualised": "visualized",+ "visualises": "visualizes",+ "visualising": "visualizing",+ "vocalisation": "vocalization",+ "vocalisations": "vocalizations",+ "vocalise": "vocalize",+ "vocalised": "vocalized",+ "vocalises": "vocalizes",+ "vocalising": "vocalizing",+ "vulcanised": "vulcanized",+ "vulgarisation": "vulgarization",+ "vulgarise": "vulgarize",+ "vulgarised": "vulgarized",+ "vulgarises": "vulgarizes",+ "vulgarising": "vulgarizing",+ "waggon": "wagon",+ "waggons": "wagons",+ "watercolour": "watercolor",+ "watercolours": "watercolors",+ "weaselled": "weaseled",+ "weaselling": "weaseling",+ "westernisation": "westernization",+ "westernise": "westernize",+ "westernised": "westernized",+ "westernises": "westernizes",+ "westernising": "westernizing",+ "womanise": "womanize",+ "womanised": "womanized",+ "womaniser": "womanizer",+ "womanisers": "womanizers",+ "womanises": "womanizes",+ "womanising": "womanizing",+ "woollen": "woolen",+ "woollens": "woolens",+ "woollies": "woolies",+ "woolly": "wooly",+ "worshipped": "worshiped",+ "worshipping": "worshiping",+ "worshipper": "worshiper",+ "yodelled": "yodeled",+ "yodelling": "yodeling",+ "yoghourt": "yogurt",+ "yoghourts": "yogurts",+ "yoghurt": "yogurt",+ "yoghurts": "yogurts",+ "mhm": "hmm",+ "mmm": "hmm"+}\ No newline at end of filediff --git a/funasr/models/sense_voice/whisper_lib/normalizers/english.py b/funasr/models/sense_voice/whisper_lib/normalizers/english.pynew file mode 100644index 000000000..cb4653098--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/normalizers/english.py@@ -0,0 +1,546 @@+import json+import os+import re+from fractions import Fraction+from typing import Iterator, List, Match, Optional, Union++from more_itertools import windowed++from .basic import remove_symbols_and_diacritics+++class EnglishNumberNormalizer:+ """+ Convert any spelled-out numbers into arabic numbers, while handling:++ - remove any commas+ - keep the suffixes such as: `1960s`, `274th`, `32nd`, etc.+ - spell out currency symbols after the number. e.g. `$20 million` -> `20000000 dollars`+ - spell out `one` and `ones`+ - interpret successive single-digit numbers as nominal: `one oh one` -> `101`+ """++ def __init__(self):+ super().__init__()++ self.zeros = {"o", "oh", "zero"}+ self.ones = {+ name: i+ for i, name in enumerate(+ [+ "one",+ "two",+ "three",+ "four",+ "five",+ "six",+ "seven",+ "eight",+ "nine",+ "ten",+ "eleven",+ "twelve",+ "thirteen",+ "fourteen",+ "fifteen",+ "sixteen",+ "seventeen",+ "eighteen",+ "nineteen",+ ],+ start=1,+ )+ }+ self.ones_plural = {+ "sixes" if name == "six" else name + "s": (value, "s")+ for name, value in self.ones.items()+ }+ self.ones_ordinal = {+ "zeroth": (0, "th"),+ "first": (1, "st"),+ "second": (2, "nd"),+ "third": (3, "rd"),+ "fifth": (5, "th"),+ "twelfth": (12, "th"),+ **{+ name + ("h" if name.endswith("t") else "th"): (value, "th")+ for name, value in self.ones.items()+ if value > 3 and value != 5 and value != 12+ },+ }+ self.ones_suffixed = {**self.ones_plural, **self.ones_ordinal}++ self.tens = {+ "twenty": 20,+ "thirty": 30,+ "forty": 40,+ "fifty": 50,+ "sixty": 60,+ "seventy": 70,+ "eighty": 80,+ "ninety": 90,+ }+ self.tens_plural = {+ name.replace("y", "ies"): (value, "s") for name, value in self.tens.items()+ }+ self.tens_ordinal = {+ name.replace("y", "ieth"): (value, "th") for name, value in self.tens.items()+ }+ self.tens_suffixed = {**self.tens_plural, **self.tens_ordinal}++ self.multipliers = {+ "hundred": 100,+ "thousand": 1_000,+ "million": 1_000_000,+ "billion": 1_000_000_000,+ "trillion": 1_000_000_000_000,+ "quadrillion": 1_000_000_000_000_000,+ "quintillion": 1_000_000_000_000_000_000,+ "sextillion": 1_000_000_000_000_000_000_000,+ "septillion": 1_000_000_000_000_000_000_000_000,+ "octillion": 1_000_000_000_000_000_000_000_000_000,+ "nonillion": 1_000_000_000_000_000_000_000_000_000_000,+ "decillion": 1_000_000_000_000_000_000_000_000_000_000_000,+ }+ self.multipliers_plural = {+ name + "s": (value, "s") for name, value in self.multipliers.items()+ }+ self.multipliers_ordinal = {+ name + "th": (value, "th") for name, value in self.multipliers.items()+ }+ self.multipliers_suffixed = {+ **self.multipliers_plural,+ **self.multipliers_ordinal,+ }+ self.decimals = {*self.ones, *self.tens, *self.zeros}++ self.preceding_prefixers = {+ "minus": "-",+ "negative": "-",+ "plus": "+",+ "positive": "+",+ }+ self.following_prefixers = {+ "pound": "£",+ "pounds": "£",+ "euro": "€",+ "euros": "€",+ "dollar": "$",+ "dollars": "$",+ "cent": "¢",+ "cents": "¢",+ }+ self.prefixes = set(+ list(self.preceding_prefixers.values()) + list(self.following_prefixers.values())+ )+ self.suffixers = {+ "per": {"cent": "%"},+ "percent": "%",+ }+ self.specials = {"and", "double", "triple", "point"}++ self.words = set(+ [+ key+ for mapping in [+ self.zeros,+ self.ones,+ self.ones_suffixed,+ self.tens,+ self.tens_suffixed,+ self.multipliers,+ self.multipliers_suffixed,+ self.preceding_prefixers,+ self.following_prefixers,+ self.suffixers,+ self.specials,+ ]+ for key in mapping+ ]+ )+ self.literal_words = {"one", "ones"}++ def process_words(self, words: List[str]) -> Iterator[str]:+ prefix: Optional[str] = None+ value: Optional[Union[str, int]] = None+ skip = False++ def to_fraction(s: str):+ try:+ return Fraction(s)+ except ValueError:+ return None++ def output(result: Union[str, int]):+ nonlocal prefix, value+ result = str(result)+ if prefix is not None:+ result = prefix + result+ value = None+ prefix = None+ return result++ if len(words) == 0:+ return++ for prev, current, next in windowed([None] + words + [None], 3):+ if skip:+ skip = False+ continue++ next_is_numeric = next is not None and re.match(r"^\d+(\.\d+)?$", next)+ has_prefix = current[0] in self.prefixes+ current_without_prefix = current[1:] if has_prefix else current+ if re.match(r"^\d+(\.\d+)?$", current_without_prefix):+ # arabic numbers (potentially with signs and fractions)+ f = to_fraction(current_without_prefix)+ assert f is not None+ if value is not None:+ if isinstance(value, str) and value.endswith("."):+ # concatenate decimals / ip address components+ value = str(value) + str(current)+ continue+ else:+ yield output(value)++ prefix = current[0] if has_prefix else prefix+ if f.denominator == 1:+ value = f.numerator # store integers as int+ else:+ value = current_without_prefix+ elif current not in self.words:+ # non-numeric words+ if value is not None:+ yield output(value)+ yield output(current)+ elif current in self.zeros:+ value = str(value or "") + "0"+ elif current in self.ones:+ ones = self.ones[current]++ if value is None:+ value = ones+ elif isinstance(value, str) or prev in self.ones:+ if prev in self.tens and ones < 10: # replace the last zero with the digit+ assert value[-1] == "0"+ value = value[:-1] + str(ones)+ else:+ value = str(value) + str(ones)+ elif ones < 10:+ if value % 10 == 0:+ value += ones+ else:+ value = str(value) + str(ones)+ else: # eleven to nineteen+ if value % 100 == 0:+ value += ones+ else:+ value = str(value) + str(ones)+ elif current in self.ones_suffixed:+ # ordinal or cardinal; yield the number right away+ ones, suffix = self.ones_suffixed[current]+ if value is None:+ yield output(str(ones) + suffix)+ elif isinstance(value, str) or prev in self.ones:+ if prev in self.tens and ones < 10:+ assert value[-1] == "0"+ yield output(value[:-1] + str(ones) + suffix)+ else:+ yield output(str(value) + str(ones) + suffix)+ elif ones < 10:+ if value % 10 == 0:+ yield output(str(value + ones) + suffix)+ else:+ yield output(str(value) + str(ones) + suffix)+ else: # eleven to nineteen+ if value % 100 == 0:+ yield output(str(value + ones) + suffix)+ else:+ yield output(str(value) + str(ones) + suffix)+ value = None+ elif current in self.tens:+ tens = self.tens[current]+ if value is None:+ value = tens+ elif isinstance(value, str):+ value = str(value) + str(tens)+ else:+ if value % 100 == 0:+ value += tens+ else:+ value = str(value) + str(tens)+ elif current in self.tens_suffixed:+ # ordinal or cardinal; yield the number right away+ tens, suffix = self.tens_suffixed[current]+ if value is None:+ yield output(str(tens) + suffix)+ elif isinstance(value, str):+ yield output(str(value) + str(tens) + suffix)+ else:+ if value % 100 == 0:+ yield output(str(value + tens) + suffix)+ else:+ yield output(str(value) + str(tens) + suffix)+ elif current in self.multipliers:+ multiplier = self.multipliers[current]+ if value is None:+ value = multiplier+ elif isinstance(value, str) or value == 0:+ f = to_fraction(value)+ p = f * multiplier if f is not None else None+ if f is not None and p.denominator == 1:+ value = p.numerator+ else:+ yield output(value)+ value = multiplier+ else:+ before = value // 1000 * 1000+ residual = value % 1000+ value = before + residual * multiplier+ elif current in self.multipliers_suffixed:+ multiplier, suffix = self.multipliers_suffixed[current]+ if value is None:+ yield output(str(multiplier) + suffix)+ elif isinstance(value, str):+ f = to_fraction(value)+ p = f * multiplier if f is not None else None+ if f is not None and p.denominator == 1:+ yield output(str(p.numerator) + suffix)+ else:+ yield output(value)+ yield output(str(multiplier) + suffix)+ else: # int+ before = value // 1000 * 1000+ residual = value % 1000+ value = before + residual * multiplier+ yield output(str(value) + suffix)+ value = None+ elif current in self.preceding_prefixers:+ # apply prefix (positive, minus, etc.) if it precedes a number+ if value is not None:+ yield output(value)++ if next in self.words or next_is_numeric:+ prefix = self.preceding_prefixers[current]+ else:+ yield output(current)+ elif current in self.following_prefixers:+ # apply prefix (dollars, cents, etc.) only after a number+ if value is not None:+ prefix = self.following_prefixers[current]+ yield output(value)+ else:+ yield output(current)+ elif current in self.suffixers:+ # apply suffix symbols (percent -> '%')+ if value is not None:+ suffix = self.suffixers[current]+ if isinstance(suffix, dict):+ if next in suffix:+ yield output(str(value) + suffix[next])+ skip = True+ else:+ yield output(value)+ yield output(current)+ else:+ yield output(str(value) + suffix)+ else:+ yield output(current)+ elif current in self.specials:+ if next not in self.words and not next_is_numeric:+ # apply special handling only if the next word can be numeric+ if value is not None:+ yield output(value)+ yield output(current)+ elif current == "and":+ # ignore "and" after hundreds, thousands, etc.+ if prev not in self.multipliers:+ if value is not None:+ yield output(value)+ yield output(current)+ elif current == "double" or current == "triple":+ if next in self.ones or next in self.zeros:+ repeats = 2 if current == "double" else 3+ ones = self.ones.get(next, 0)+ value = str(value or "") + str(ones) * repeats+ skip = True+ else:+ if value is not None:+ yield output(value)+ yield output(current)+ elif current == "point":+ if next in self.decimals or next_is_numeric:+ value = str(value or "") + "."+ else:+ # should all have been covered at this point+ raise ValueError(f"Unexpected token: {current}")+ else:+ # all should have been covered at this point+ raise ValueError(f"Unexpected token: {current}")++ if value is not None:+ yield output(value)++ def preprocess(self, s: str):+ # replace "<number> and a half" with "<number> point five"+ results = []++ segments = re.split(r"\band\s+a\s+half\b", s)+ for i, segment in enumerate(segments):+ if len(segment.strip()) == 0:+ continue+ if i == len(segments) - 1:+ results.append(segment)+ else:+ results.append(segment)+ last_word = segment.rsplit(maxsplit=2)[-1]+ if last_word in self.decimals or last_word in self.multipliers:+ results.append("point five")+ else:+ results.append("and a half")++ s = " ".join(results)++ # put a space at number/letter boundary+ s = re.sub(r"([a-z])([0-9])", r"\1 \2", s)+ s = re.sub(r"([0-9])([a-z])", r"\1 \2", s)++ # but remove spaces which could be a suffix+ s = re.sub(r"([0-9])\s+(st|nd|rd|th|s)\b", r"\1\2", s)++ return s++ def postprocess(self, s: str):+ def combine_cents(m: Match):+ try:+ currency = m.group(1)+ integer = m.group(2)+ cents = int(m.group(3))+ return f"{currency}{integer}.{cents:02d}"+ except ValueError:+ return m.string++ def extract_cents(m: Match):+ try:+ return f"¢{int(m.group(1))}"+ except ValueError:+ return m.string++ # apply currency postprocessing; "$2 and ¢7" -> "$2.07"+ s = re.sub(r"([€£$])([0-9]+) (?:and )?¢([0-9]{1,2})\b", combine_cents, s)+ s = re.sub(r"[€£$]0.([0-9]{1,2})\b", extract_cents, s)++ # write "one(s)" instead of "1(s)", just for the readability+ s = re.sub(r"\b1(s?)\b", r"one\1", s)++ return s++ def __call__(self, s: str):+ s = self.preprocess(s)+ s = " ".join(word for word in self.process_words(s.split()) if word is not None)+ s = self.postprocess(s)++ return s+++class EnglishSpellingNormalizer:+ """+ Applies British-American spelling mappings as listed in [1].++ [1] https://www.tysto.com/uk-us-spelling-list.html+ """++ def __init__(self):+ mapping_path = os.path.join(os.path.dirname(__file__), "english.json")+ self.mapping = json.load(open(mapping_path))++ def __call__(self, s: str):+ return " ".join(self.mapping.get(word, word) for word in s.split())+++class EnglishTextNormalizer:+ def __init__(self):+ self.ignore_patterns = r"\b(hmm|mm|mhm|mmm|uh|um)\b"+ self.replacers = {+ # common contractions+ r"\bwon't\b": "will not",+ r"\bcan't\b": "can not",+ r"\blet's\b": "let us",+ r"\bain't\b": "aint",+ r"\by'all\b": "you all",+ r"\bwanna\b": "want to",+ r"\bgotta\b": "got to",+ r"\bgonna\b": "going to",+ r"\bi'ma\b": "i am going to",+ r"\bimma\b": "i am going to",+ r"\bwoulda\b": "would have",+ r"\bcoulda\b": "could have",+ r"\bshoulda\b": "should have",+ r"\bma'am\b": "madam",+ # contractions in titles/prefixes+ r"\bmr\b": "mister ",+ r"\bmrs\b": "missus ",+ r"\bst\b": "saint ",+ r"\bdr\b": "doctor ",+ r"\bprof\b": "professor ",+ r"\bcapt\b": "captain ",+ r"\bgov\b": "governor ",+ r"\bald\b": "alderman ",+ r"\bgen\b": "general ",+ r"\bsen\b": "senator ",+ r"\brep\b": "representative ",+ r"\bpres\b": "president ",+ r"\brev\b": "reverend ",+ r"\bhon\b": "honorable ",+ r"\basst\b": "assistant ",+ r"\bassoc\b": "associate ",+ r"\blt\b": "lieutenant ",+ r"\bcol\b": "colonel ",+ r"\bjr\b": "junior ",+ r"\bsr\b": "senior ",+ r"\besq\b": "esquire ",+ # prefect tenses, ideally it should be any past participles, but it's harder..+ r"'d been\b": " had been",+ r"'s been\b": " has been",+ r"'d gone\b": " had gone",+ r"'s gone\b": " has gone",+ r"'d done\b": " had done", # "'s done" is ambiguous+ r"'s got\b": " has got",+ # general contractions+ r"n't\b": " not",+ r"'re\b": " are",+ r"'s\b": " is",+ r"'d\b": " would",+ r"'ll\b": " will",+ r"'t\b": " not",+ r"'ve\b": " have",+ r"'m\b": " am",+ }+ self.standardize_numbers = EnglishNumberNormalizer()+ self.standardize_spellings = EnglishSpellingNormalizer()++ def __call__(self, s: str):+ s = s.lower()++ s = re.sub(r"[<\[][^>\]]*[>\]]", "", s) # remove words between brackets+ s = re.sub(r"\(([^)]+?)\)", "", s) # remove words between parenthesis+ s = re.sub(self.ignore_patterns, "", s)+ s = re.sub(r"\s+'", "'", s) # when there's a space before an apostrophe++ for pattern, replacement in self.replacers.items():+ s = re.sub(pattern, replacement, s)++ s = re.sub(r"(\d),(\d)", r"\1\2", s) # remove commas between digits+ s = re.sub(r"\.([^0-9]|$)", r" \1", s) # remove periods not followed by numbers+ s = remove_symbols_and_diacritics(s, keep=".%$¢€£") # keep numeric symbols++ s = self.standardize_numbers(s)+ s = self.standardize_spellings(s)++ # now remove prefix/suffix symbols that are not preceded/followed by numbers+ s = re.sub(r"[.$¢€£]([^0-9])", r" \1", s)+ s = re.sub(r"([^0-9])%", r"\1 ", s)++ s = re.sub(r"\s+", " ", s) # replace any successive whitespaces with a space++ return sdiff --git a/funasr/models/sense_voice/whisper_lib/timing.py b/funasr/models/sense_voice/whisper_lib/timing.pynew file mode 100644index 000000000..ba9cb1399--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/timing.py@@ -0,0 +1,362 @@+import itertools+import subprocess+import warnings+from dataclasses import dataclass+from typing import TYPE_CHECKING, List++import numba+import numpy as np+import torch+import torch.nn.functional as F++from .audio import HOP_LENGTH, SAMPLE_RATE, TOKENS_PER_SECOND+from .tokenizer import Tokenizer++if TYPE_CHECKING:+ from .model import Whisper+++def median_filter(x: torch.Tensor, filter_width: int):+ """Apply a median filter of width `filter_width` along the last dimension of `x`"""+ pad_width = filter_width // 2+ if x.shape[-1] <= pad_width:+ # F.pad requires the padding width to be smaller than the input dimension+ return x++ if (ndim := x.ndim) <= 2:+ # `F.pad` does not support 1D or 2D inputs for reflect padding but supports 3D and 4D+ x = x[None, None, :]++ assert filter_width > 0 and filter_width % 2 == 1, "`filter_width` should be an odd number"++ result = None+ x = F.pad(x, (filter_width // 2, filter_width // 2, 0, 0), mode="reflect")+ if x.is_cuda:+ try:+ from .triton_ops import median_filter_cuda++ result = median_filter_cuda(x, filter_width)+ except (RuntimeError, subprocess.CalledProcessError):+ warnings.warn(+ "Failed to launch Triton kernels, likely due to missing CUDA toolkit; "+ "falling back to a slower median kernel implementation..."+ )++ if result is None:+ # sort() is faster than torch.median (https://github.com/pytorch/pytorch/issues/51450)+ result = x.unfold(-1, filter_width, 1).sort()[0][..., filter_width // 2]++ if ndim <= 2:+ result = result[0, 0]++ return result++[email protected](nopython=True)+def backtrace(trace: np.ndarray):+ i = trace.shape[0] - 1+ j = trace.shape[1] - 1+ trace[0, :] = 2+ trace[:, 0] = 1++ result = []+ while i > 0 or j > 0:+ result.append((i - 1, j - 1))++ if trace[i, j] == 0:+ i -= 1+ j -= 1+ elif trace[i, j] == 1:+ i -= 1+ elif trace[i, j] == 2:+ j -= 1+ else:+ raise ValueError("Unexpected trace[i, j]")++ result = np.array(result)+ return result[::-1, :].T++[email protected](nopython=True, parallel=True)+def dtw_cpu(x: np.ndarray):+ N, M = x.shape+ cost = np.ones((N + 1, M + 1), dtype=np.float32) * np.inf+ trace = -np.ones((N + 1, M + 1), dtype=np.float32)++ cost[0, 0] = 0+ for j in range(1, M + 1):+ for i in range(1, N + 1):+ c0 = cost[i - 1, j - 1]+ c1 = cost[i - 1, j]+ c2 = cost[i, j - 1]++ if c0 < c1 and c0 < c2:+ c, t = c0, 0+ elif c1 < c0 and c1 < c2:+ c, t = c1, 1+ else:+ c, t = c2, 2++ cost[i, j] = x[i - 1, j - 1] + c+ trace[i, j] = t++ return backtrace(trace)+++def dtw_cuda(x, BLOCK_SIZE=1024):+ from .triton_ops import dtw_kernel++ M, N = x.shape+ assert M < BLOCK_SIZE, f"M should be smaller than {BLOCK_SIZE=}"++ x_skew = F.pad(x, (0, M + 1), value=np.inf).flatten()[: M * (N + M)].reshape(M, N + M)+ x_skew = x_skew.T.contiguous()+ cost = torch.ones(N + M + 2, M + 2) * np.inf+ cost[0, 0] = 0+ cost = cost.cuda()+ trace = torch.zeros_like(cost, dtype=torch.int32)++ dtw_kernel[(1,)](+ cost,+ trace,+ x_skew,+ x_skew.stride(0),+ cost.stride(0),+ trace.stride(0),+ N,+ M,+ BLOCK_SIZE=BLOCK_SIZE,+ )++ trace = trace.T.flatten()[: (M + 1) * (M + N + 3)].reshape(M + 1, M + N + 3)[:, : N + 1]+ return backtrace(trace.cpu().numpy())+++def dtw(x: torch.Tensor) -> np.ndarray:+ if x.is_cuda:+ try:+ return dtw_cuda(x)+ except (RuntimeError, subprocess.CalledProcessError):+ warnings.warn(+ "Failed to launch Triton kernels, likely due to missing CUDA toolkit; "+ "falling back to a slower DTW implementation..."+ )++ return dtw_cpu(x.double().cpu().numpy())+++@dataclass+class WordTiming:+ word: str+ tokens: List[int]+ start: float+ end: float+ probability: float+++def find_alignment(+ model: "Whisper",+ tokenizer: Tokenizer,+ text_tokens: List[int],+ mel: torch.Tensor,+ num_frames: int,+ *,+ medfilt_width: int = 7,+ qk_scale: float = 1.0,+) -> List[WordTiming]:+ if len(text_tokens) == 0:+ return []++ tokens = torch.tensor(+ [+ *tokenizer.sot_sequence,+ tokenizer.no_timestamps,+ *text_tokens,+ tokenizer.eot,+ ]+ ).to(model.device)++ # install hooks on the cross attention layers to retrieve the attention weights+ QKs = [None] * model.dims.n_text_layer+ hooks = [+ block.cross_attn.register_forward_hook(+ lambda _, ins, outs, index=i: QKs.__setitem__(index, outs[-1][0])+ )+ for i, block in enumerate(model.decoder.blocks)+ ]++ with torch.no_grad():+ logits = model(mel.unsqueeze(0), tokens.unsqueeze(0))[0]+ sampled_logits = logits[len(tokenizer.sot_sequence) :, : tokenizer.eot]+ token_probs = sampled_logits.softmax(dim=-1)+ text_token_probs = token_probs[np.arange(len(text_tokens)), text_tokens]+ text_token_probs = text_token_probs.tolist()++ for hook in hooks:+ hook.remove()++ # heads * tokens * frames+ weights = torch.stack([QKs[_l][_h] for _l, _h in model.alignment_heads.indices().T])+ weights = weights[:, :, : num_frames // 2]+ weights = (weights * qk_scale).softmax(dim=-1)+ std, mean = torch.std_mean(weights, dim=-2, keepdim=True, unbiased=False)+ weights = (weights - mean) / std+ weights = median_filter(weights, medfilt_width)++ matrix = weights.mean(axis=0)+ matrix = matrix[len(tokenizer.sot_sequence) : -1]+ text_indices, time_indices = dtw(-matrix)++ words, word_tokens = tokenizer.split_to_word_tokens(text_tokens + [tokenizer.eot])+ if len(word_tokens) <= 1:+ # return on eot only+ # >>> np.pad([], (1, 0))+ # array([0.])+ # This results in crashes when we lookup jump_times with float, like+ # IndexError: arrays used as indices must be of integer (or boolean) type+ return []+ word_boundaries = np.pad(np.cumsum([len(t) for t in word_tokens[:-1]]), (1, 0))++ jumps = np.pad(np.diff(text_indices), (1, 0), constant_values=1).astype(bool)+ jump_times = time_indices[jumps] / TOKENS_PER_SECOND+ start_times = jump_times[word_boundaries[:-1]]+ end_times = jump_times[word_boundaries[1:]]+ word_probabilities = [+ np.mean(text_token_probs[i:j]) for i, j in zip(word_boundaries[:-1], word_boundaries[1:])+ ]++ return [+ WordTiming(word, tokens, start, end, probability)+ for word, tokens, start, end, probability in zip(+ words, word_tokens, start_times, end_times, word_probabilities+ )+ ]+++def merge_punctuations(alignment: List[WordTiming], prepended: str, appended: str):+ # merge prepended punctuations+ i = len(alignment) - 2+ j = len(alignment) - 1+ while i >= 0:+ previous = alignment[i]+ following = alignment[j]+ if previous.word.startswith(" ") and previous.word.strip() in prepended:+ # prepend it to the following word+ following.word = previous.word + following.word+ following.tokens = previous.tokens + following.tokens+ previous.word = ""+ previous.tokens = []+ else:+ j = i+ i -= 1++ # merge appended punctuations+ i = 0+ j = 1+ while j < len(alignment):+ previous = alignment[i]+ following = alignment[j]+ if not previous.word.endswith(" ") and following.word in appended:+ # append it to the previous word+ previous.word = previous.word + following.word+ previous.tokens = previous.tokens + following.tokens+ following.word = ""+ following.tokens = []+ else:+ i = j+ j += 1+++def add_word_timestamps(+ *,+ segments: List[dict],+ model: "Whisper",+ tokenizer: Tokenizer,+ mel: torch.Tensor,+ num_frames: int,+ prepend_punctuations: str = "\"'“¿([{-",+ append_punctuations: str = "\"'.。,,!!??::”)]}、",+ last_speech_timestamp: float,+ **kwargs,+):+ if len(segments) == 0:+ return++ text_tokens_per_segment = [+ [token for token in segment["tokens"] if token < tokenizer.eot] for segment in segments+ ]++ text_tokens = list(itertools.chain.from_iterable(text_tokens_per_segment))+ alignment = find_alignment(model, tokenizer, text_tokens, mel, num_frames, **kwargs)+ word_durations = np.array([t.end - t.start for t in alignment])+ word_durations = word_durations[word_durations.nonzero()]+ median_duration = np.median(word_durations) if len(word_durations) > 0 else 0.0+ median_duration = min(0.7, float(median_duration))+ max_duration = median_duration * 2++ # hack: truncate long words at sentence boundaries.+ # a better segmentation algorithm based on VAD should be able to replace this.+ if len(word_durations) > 0:+ sentence_end_marks = ".。!!??"+ # ensure words at sentence boundaries are not longer than twice the median word duration.+ for i in range(1, len(alignment)):+ if alignment[i].end - alignment[i].start > max_duration:+ if alignment[i].word in sentence_end_marks:+ alignment[i].end = alignment[i].start + max_duration+ elif alignment[i - 1].word in sentence_end_marks:+ alignment[i].start = alignment[i].end - max_duration++ merge_punctuations(alignment, prepend_punctuations, append_punctuations)++ time_offset = segments[0]["seek"] * HOP_LENGTH / SAMPLE_RATE+ word_index = 0++ for segment, text_tokens in zip(segments, text_tokens_per_segment):+ saved_tokens = 0+ words = []++ while word_index < len(alignment) and saved_tokens < len(text_tokens):+ timing = alignment[word_index]++ if timing.word:+ words.append(+ dict(+ word=timing.word,+ start=round(time_offset + timing.start, 2),+ end=round(time_offset + timing.end, 2),+ probability=timing.probability,+ )+ )++ saved_tokens += len(timing.tokens)+ word_index += 1++ # hack: truncate long words at segment boundaries.+ # a better segmentation algorithm based on VAD should be able to replace this.+ if len(words) > 0:+ # ensure the first and second word after a pause is not longer than+ # twice the median word duration.+ if words[0]["end"] - last_speech_timestamp > median_duration * 4 and (+ words[0]["end"] - words[0]["start"] > max_duration+ or (len(words) > 1 and words[1]["end"] - words[0]["start"] > max_duration * 2)+ ):+ if len(words) > 1 and words[1]["end"] - words[1]["start"] > max_duration:+ boundary = max(words[1]["end"] / 2, words[1]["end"] - max_duration)+ words[0]["end"] = words[1]["start"] = boundary+ words[0]["start"] = max(0, words[0]["end"] - max_duration)++ # prefer the segment-level start timestamp if the first word is too long.+ if segment["start"] < words[0]["end"] and segment["start"] - 0.5 > words[0]["start"]:+ words[0]["start"] = max(0, min(words[0]["end"] - median_duration, segment["start"]))+ else:+ segment["start"] = words[0]["start"]++ # prefer the segment-level end timestamp if the last word is too long.+ if segment["end"] > words[-1]["start"] and segment["end"] + 0.5 < words[-1]["end"]:+ words[-1]["end"] = max(words[-1]["start"] + median_duration, segment["end"])+ else:+ segment["end"] = words[-1]["end"]++ last_speech_timestamp = segment["end"]++ segment["words"] = wordsdiff --git a/funasr/models/sense_voice/whisper_lib/tokenizer.py b/funasr/models/sense_voice/whisper_lib/tokenizer.pynew file mode 100644index 000000000..5b276c2a0--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/tokenizer.py@@ -0,0 +1,456 @@+import base64+import os+import string+from dataclasses import dataclass, field+from functools import cached_property, lru_cache+from typing import Dict, List, Optional, Tuple++import tiktoken++# FIX(funasr): sense vocie+LANGUAGES = {+ "en": "english",+ "zh": "chinese",+ "de": "german",+ "es": "spanish",+ "ru": "russian",+ "ko": "korean",+ "fr": "french",+ "ja": "japanese",+ "pt": "portuguese",+ "tr": "turkish",+ "pl": "polish",+ "ca": "catalan",+ "nl": "dutch",+ "ar": "arabic",+ "sv": "swedish",+ "it": "italian",+ "id": "indonesian",+ "hi": "hindi",+ "fi": "finnish",+ "vi": "vietnamese",+ "he": "hebrew",+ "uk": "ukrainian",+ "el": "greek",+ "ms": "malay",+ "cs": "czech",+ "ro": "romanian",+ "da": "danish",+ "hu": "hungarian",+ "ta": "tamil",+ "no": "norwegian",+ "th": "thai",+ "ur": "urdu",+ "hr": "croatian",+ "bg": "bulgarian",+ "lt": "lithuanian",+ "la": "latin",+ "mi": "maori",+ "ml": "malayalam",+ "cy": "welsh",+ "sk": "slovak",+ "te": "telugu",+ "fa": "persian",+ "lv": "latvian",+ "bn": "bengali",+ "sr": "serbian",+ "az": "azerbaijani",+ "sl": "slovenian",+ "kn": "kannada",+ "et": "estonian",+ "mk": "macedonian",+ "br": "breton",+ "eu": "basque",+ "is": "icelandic",+ "hy": "armenian",+ "ne": "nepali",+ "mn": "mongolian",+ "bs": "bosnian",+ "kk": "kazakh",+ "sq": "albanian",+ "sw": "swahili",+ "gl": "galician",+ "mr": "marathi",+ "pa": "punjabi",+ "si": "sinhala",+ "km": "khmer",+ "sn": "shona",+ "yo": "yoruba",+ "so": "somali",+ "af": "afrikaans",+ "oc": "occitan",+ "ka": "georgian",+ "be": "belarusian",+ "tg": "tajik",+ "sd": "sindhi",+ "gu": "gujarati",+ "am": "amharic",+ "yi": "yiddish",+ "lo": "lao",+ "uz": "uzbek",+ "fo": "faroese",+ "ht": "haitian creole",+ "ps": "pashto",+ "tk": "turkmen",+ "nn": "nynorsk",+ "mt": "maltese",+ "sa": "sanskrit",+ "lb": "luxembourgish",+ "my": "myanmar",+ "bo": "tibetan",+ "tl": "tagalog",+ "mg": "malagasy",+ "as": "assamese",+ "tt": "tatar",+ "haw": "hawaiian",+ "ln": "lingala",+ "ha": "hausa",+ "ba": "bashkir",+ "jw": "javanese",+ "su": "sundanese",+ "yue": "cantonese",+ "minnan": "minnan",+ "wuyu": "wuyu",+ "dialect": "dialect",+ "zh/en": "zh/en",+ "en/zh": "en/zh",+}++# language code lookup by name, with a few language aliases+TO_LANGUAGE_CODE = {+ **{language: code for code, language in LANGUAGES.items()},+ "burmese": "my",+ "valencian": "ca",+ "flemish": "nl",+ "haitian": "ht",+ "letzeburgesch": "lb",+ "pushto": "ps",+ "panjabi": "pa",+ "moldavian": "ro",+ "moldovan": "ro",+ "sinhalese": "si",+ "castilian": "es",+ "mandarin": "zh",+}++# FIX(funasr): sense vocie+AUDIO_EVENT = {+ "ASR": "ASR",+ "AED": "AED",+ "SER": "SER",+ "Speech": "Speech",+ "/Speech": "/Speech",+ "BGM": "BGM",+ "/BGM": "/BGM",+ "Laughter": "Laughter",+ "/Laughter": "/Laughter",+ "Applause": "Applause",+ "/Applause": "/Applause",+}++EMOTION = {+ "HAPPY": "HAPPY",+ "SAD": "SAD",+ "ANGRY": "ANGRY",+ "NEUTRAL": "NEUTRAL",+}+++@dataclass+class Tokenizer:+ """A thin wrapper around `tiktoken` providing quick access to special tokens"""++ encoding: tiktoken.Encoding+ num_languages: int+ language: Optional[str] = None+ task: Optional[str] = None+ sot_sequence: Tuple[int] = ()+ special_tokens: Dict[str, int] = field(default_factory=dict)++ def __post_init__(self):+ for special in self.encoding.special_tokens_set:+ special_token = self.encoding.encode_single_token(special)+ self.special_tokens[special] = special_token++ sot: int = self.special_tokens["<|startoftranscript|>"]+ translate: int = self.special_tokens["<|translate|>"]+ transcribe: int = self.special_tokens["<|transcribe|>"]++ langs = tuple(LANGUAGES.keys())[: self.num_languages]+ sot_sequence = [sot]+ if self.language is not None:+ if self.language == "nospeech":+ sot_sequence.append(self.no_speech)+ else:+ sot_sequence.append(sot + 1 + langs.index(self.language))+ # if self.language is not None:+ # sot_sequence.append(sot + 1 + langs.index(self.language))+ if self.task is not None:+ task_token: int = transcribe if self.task == "transcribe" else translate+ sot_sequence.append(task_token)++ self.sot_sequence = tuple(sot_sequence)++ def encode(self, text, **kwargs):+ return self.encoding.encode(text, **kwargs)++ def decode(self, token_ids: List[int], **kwargs) -> str:+ token_ids = [t for t in token_ids if t < self.timestamp_begin]+ return self.encoding.decode(token_ids, **kwargs)++ def decode_with_timestamps(self, token_ids: List[int], **kwargs) -> str:+ """+ Timestamp tokens are above other special tokens' id range and are ignored by `decode()`.+ This method decodes given tokens with timestamps tokens annotated, e.g. "<|1.08|>".+ """+ return self.encoding.decode(token_ids, **kwargs)++ def get_vocab_size(self) -> int:+ return self.encoding.n_vocab++ @cached_property+ def eot(self) -> int:+ return self.encoding.eot_token++ @cached_property+ def transcribe(self) -> int:+ return self.special_tokens["<|transcribe|>"]++ @cached_property+ def translate(self) -> int:+ return self.special_tokens["<|translate|>"]++ @cached_property+ def sot(self) -> int:+ return self.special_tokens["<|startoftranscript|>"]++ @cached_property+ def sot_sense(self) -> int:+ return self.special_tokens["<|startoftranscript|>"]++ @cached_property+ def sot_lm(self) -> int:+ return self.special_tokens["<|startoflm|>"]++ @cached_property+ def sot_prev(self) -> int:+ return self.special_tokens["<|startofprev|>"]++ @cached_property+ def no_speech(self) -> int:+ return self.special_tokens["<|nospeech|>"]++ @cached_property+ def no_timestamps(self) -> int:+ return self.special_tokens["<|notimestamps|>"]++ @cached_property+ def timestamp_begin(self) -> int:+ return self.special_tokens["<|0.00|>"]++ @cached_property+ def language_token(self) -> int:+ """Returns the token id corresponding to the value of the `language` field"""+ if self.language is None:+ raise ValueError("This tokenizer does not have language token configured")++ return self.to_language_token(self.language)++ def to_language_token(self, language):+ if token := self.special_tokens.get(f"<|{language}|>", None):+ return token++ raise KeyError(f"Language {language} not found in tokenizer.")++ @cached_property+ def all_language_tokens(self) -> Tuple[int]:+ result = []+ for token, token_id in self.special_tokens.items():+ if token.strip("<|>") in LANGUAGES:+ result.append(token_id)+ return tuple(result)[: self.num_languages]++ @cached_property+ def all_language_codes(self) -> Tuple[str]:+ return tuple(self.decode([_l]).strip("<|>") for _l in self.all_language_tokens)++ @cached_property+ def sot_sequence_including_notimestamps(self) -> Tuple[int]:+ return tuple(list(self.sot_sequence) + [self.no_timestamps])++ @cached_property+ def non_speech_tokens(self) -> Tuple[int]:+ """+ Returns the list of tokens to suppress in order to avoid any speaker tags or non-speech+ annotations, to prevent sampling texts that are not actually spoken in the audio, e.g.++ - ♪♪♪+ - ( SPEAKING FOREIGN LANGUAGE )+ - [DAVID] Hey there,++ keeping basic punctuations like commas, periods, question marks, exclamation points, etc.+ """+ symbols = list('"#()*+/:;<=>@[\\]^_`{|}~「」『』')+ symbols += "<< >> <<< >>> -- --- -( -[ (' (\" (( )) ((( ))) [[ ]] {{ }} ♪♪ ♪♪♪".split()++ # symbols that may be a single token or multiple tokens depending on the tokenizer.+ # In case they're multiple tokens, suppress the first token, which is safe because:+ # These are between U+2640 and U+267F miscellaneous symbols that are okay to suppress+ # in generations, and in the 3-byte UTF-8 representation they share the first two bytes.+ miscellaneous = set("♩♪♫♬♭♮♯")+ assert all(0x2640 <= ord(c) <= 0x267F for c in miscellaneous)++ # allow hyphens "-" and single quotes "'" between words, but not at the beginning of a word+ result = {self.encoding.encode(" -")[0], self.encoding.encode(" '")[0]}+ for symbol in symbols + list(miscellaneous):+ for tokens in [+ self.encoding.encode(symbol),+ self.encoding.encode(" " + symbol),+ ]:+ if len(tokens) == 1 or symbol in miscellaneous:+ result.add(tokens[0])++ return tuple(sorted(result))++ def split_to_word_tokens(self, tokens: List[int]):+ if self.language in {"zh", "ja", "th", "lo", "my", "yue"}:+ # These languages don't typically use spaces, so it is difficult to split words+ # without morpheme analysis. Here, we instead split words at any+ # position where the tokens are decoded as valid unicode points+ return self.split_tokens_on_unicode(tokens)++ return self.split_tokens_on_spaces(tokens)++ def split_tokens_on_unicode(self, tokens: List[int]):+ decoded_full = self.decode_with_timestamps(tokens)+ replacement_char = "\ufffd"++ words = []+ word_tokens = []+ current_tokens = []+ unicode_offset = 0++ for token in tokens:+ current_tokens.append(token)+ decoded = self.decode_with_timestamps(current_tokens)++ if (+ replacement_char not in decoded+ or decoded_full[unicode_offset + decoded.index(replacement_char)]+ == replacement_char+ ):+ words.append(decoded)+ word_tokens.append(current_tokens)+ current_tokens = []+ unicode_offset += len(decoded)++ return words, word_tokens++ def split_tokens_on_spaces(self, tokens: List[int]):+ subwords, subword_tokens_list = self.split_tokens_on_unicode(tokens)+ words = []+ word_tokens = []++ for subword, subword_tokens in zip(subwords, subword_tokens_list):+ special = subword_tokens[0] >= self.eot+ with_space = subword.startswith(" ")+ punctuation = subword.strip() in string.punctuation+ if special or with_space or punctuation or len(words) == 0:+ words.append(subword)+ word_tokens.append(subword_tokens)+ else:+ words[-1] = words[-1] + subword+ word_tokens[-1].extend(subword_tokens)++ return words, word_tokens+++@lru_cache(maxsize=None)+def get_encoding(name: str = "gpt2", num_languages: int = 99, vocab_path: str = None):+ if vocab_path is None:+ vocab_path = os.path.join(os.path.dirname(__file__), "assets", f"{name}.tiktoken")++ ranks = {+ base64.b64decode(token): int(rank)+ for token, rank in (line.split() for line in open(vocab_path) if line)+ }+ n_vocab = len(ranks)+ special_tokens = {}++ if False: # name == "gpt2" or name == "multilingual":+ specials = [+ "<|endoftext|>",+ "<|startoftranscript|>",+ *[f"<|{lang}|>" for lang in list(LANGUAGES.keys())[:num_languages]],+ "<|translate|>",+ "<|transcribe|>",+ "<|startoflm|>",+ "<|startofprev|>",+ "<|nospeech|>",+ "<|notimestamps|>",+ *[f"<|{i * 0.02:.2f}|>" for i in range(1501)],+ ]+ else:+ specials = [+ "<|endoftext|>",+ "<|startoftranscript|>",+ *[f"<|{lang}|>" for lang in list(LANGUAGES.keys())[:num_languages]],+ *[f"<|{audio_event}|>" for audio_event in list(AUDIO_EVENT.keys())],+ *[f"<|{emotion}|>" for emotion in list(EMOTION.keys())],+ "<|translate|>",+ "<|transcribe|>",+ "<|startoflm|>",+ "<|startofprev|>",+ "<|nospeech|>",+ "<|notimestamps|>",+ *[f"<|SPECIAL_TOKEN_{i}|>" for i in range(1, 51)],+ *[f"<|{i * 0.02:.2f}|>" for i in range(1501)],+ ]++ for token in specials:+ special_tokens[token] = n_vocab+ n_vocab += 1++ return tiktoken.Encoding(+ name=os.path.basename(vocab_path),+ explicit_n_vocab=n_vocab,+ pat_str=r"""'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""",+ mergeable_ranks=ranks,+ special_tokens=special_tokens,+ )+++@lru_cache(maxsize=None)+def get_tokenizer(+ multilingual: bool,+ *,+ num_languages: int = 99,+ language: Optional[str] = None,+ task: Optional[str] = None, # Literal["transcribe", "translate", None]+ encoding_path: Optional[str] = None,+ vocab_path: Optional[str] = None,+) -> Tokenizer:+ if language is not None:+ language = language.lower()+ if language not in LANGUAGES:+ if language in TO_LANGUAGE_CODE:+ language = TO_LANGUAGE_CODE[language]+ elif language == "nospeech":+ pass+ else:+ raise ValueError(f"Unsupported language: {language}")++ if multilingual:+ encoding_name = "multilingual"+ language = language or "en"+ task = task or "transcribe"+ else:+ encoding_name = "gpt2"+ language = None+ task = None+ if encoding_path is not None:+ encoding_name = encoding_path++ encoding = get_encoding(name=encoding_name, num_languages=num_languages, vocab_path=vocab_path)++ return Tokenizer(encoding=encoding, num_languages=num_languages, language=language, task=task)diff --git a/funasr/models/sense_voice/whisper_lib/transcribe.py b/funasr/models/sense_voice/whisper_lib/transcribe.pynew file mode 100644index 000000000..5c5f49d80--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/transcribe.py@@ -0,0 +1,573 @@+import argparse+import os+import traceback+import warnings+from typing import TYPE_CHECKING, List, Optional, Tuple, Union++import numpy as np+import torch+import tqdm++from .audio import (+ FRAMES_PER_SECOND,+ HOP_LENGTH,+ N_FRAMES,+ N_SAMPLES,+ SAMPLE_RATE,+ log_mel_spectrogram,+ pad_or_trim,+)+from .decoding import DecodingOptions, DecodingResult+from .timing import add_word_timestamps+from .tokenizer import LANGUAGES, TO_LANGUAGE_CODE, get_tokenizer+from .utils import (+ exact_div,+ format_timestamp,+ get_end,+ get_writer,+ make_safe,+ optional_float,+ optional_int,+ str2bool,+)++if TYPE_CHECKING:+ from .model import Whisper+++def transcribe(+ model: "Whisper",+ audio: Union[str, np.ndarray, torch.Tensor],+ *,+ verbose: Optional[bool] = None,+ temperature: Union[float, Tuple[float, ...]] = (0.0, 0.2, 0.4, 0.6, 0.8, 1.0),+ compression_ratio_threshold: Optional[float] = 2.4,+ logprob_threshold: Optional[float] = -1.0,+ no_speech_threshold: Optional[float] = 0.6,+ condition_on_previous_text: bool = True,+ initial_prompt: Optional[str] = None,+ word_timestamps: bool = False,+ prepend_punctuations: str = "\"'“¿([{-",+ append_punctuations: str = "\"'.。,,!!??::”)]}、",+ clip_timestamps: Union[str, List[float]] = "0",+ hallucination_silence_threshold: Optional[float] = None,+ **decode_options,+):+ """+ Transcribe an audio file using Whisper++ Parameters+ ----------+ model: Whisper+ The Whisper model instance++ audio: Union[str, np.ndarray, torch.Tensor]+ The path to the audio file to open, or the audio waveform++ verbose: bool+ Whether to display the text being decoded to the console. If True, displays all the details,+ If False, displays minimal details. If None, does not display anything++ temperature: Union[float, Tuple[float, ...]]+ Temperature for sampling. It can be a tuple of temperatures, which will be successively used+ upon failures according to either `compression_ratio_threshold` or `logprob_threshold`.++ compression_ratio_threshold: float+ If the gzip compression ratio is above this value, treat as failed++ logprob_threshold: float+ If the average log probability over sampled tokens is below this value, treat as failed++ no_speech_threshold: float+ If the no_speech probability is higher than this value AND the average log probability+ over sampled tokens is below `logprob_threshold`, consider the segment as silent++ condition_on_previous_text: bool+ if True, the previous output of the model is provided as a prompt for the next window;+ disabling may make the text inconsistent across windows, but the model becomes less prone to+ getting stuck in a failure loop, such as repetition looping or timestamps going out of sync.++ word_timestamps: bool+ Extract word-level timestamps using the cross-attention pattern and dynamic time warping,+ and include the timestamps for each word in each segment.++ prepend_punctuations: str+ If word_timestamps is True, merge these punctuation symbols with the next word++ append_punctuations: str+ If word_timestamps is True, merge these punctuation symbols with the previous word++ initial_prompt: Optional[str]+ Optional text to provide as a prompt for the first window. This can be used to provide, or+ "prompt-engineer" a context for transcription, e.g. custom vocabularies or proper nouns+ to make it more likely to predict those word correctly.++ decode_options: dict+ Keyword arguments to construct `DecodingOptions` instances++ clip_timestamps: Union[str, List[float]]+ Comma-separated list start,end,start,end,... timestamps (in seconds) of clips to process.+ The last end timestamp defaults to the end of the file.++ hallucination_silence_threshold: Optional[float]+ When word_timestamps is True, skip silent periods longer than this threshold (in seconds)+ when a possible hallucination is detected++ Returns+ -------+ A dictionary containing the resulting text ("text") and segment-level details ("segments"), and+ the spoken language ("language"), which is detected when `decode_options["language"]` is None.+ """+ dtype = torch.float16 if decode_options.get("fp16", True) else torch.float32+ if model.device == torch.device("cpu"):+ if torch.cuda.is_available():+ warnings.warn("Performing inference on CPU when CUDA is available")+ if dtype == torch.float16:+ warnings.warn("FP16 is not supported on CPU; using FP32 instead")+ dtype = torch.float32++ if dtype == torch.float32:+ decode_options["fp16"] = False++ # Pad 30-seconds of silence to the input audio, for slicing+ mel = log_mel_spectrogram(audio, model.dims.n_mels, padding=N_SAMPLES)+ content_frames = mel.shape[-1] - N_FRAMES+ content_duration = float(content_frames * HOP_LENGTH / SAMPLE_RATE)++ if decode_options.get("language", None) is None:+ if not model.is_multilingual:+ decode_options["language"] = "en"+ else:+ if verbose:+ print(+ "Detecting language using up to the first 30 seconds. Use `--language` to specify the language"+ )+ mel_segment = pad_or_trim(mel, N_FRAMES).to(model.device).to(dtype)+ _, probs = model.detect_language(mel_segment)+ decode_options["language"] = max(probs, key=probs.get)+ if verbose is not None:+ print(f"Detected language: {LANGUAGES[decode_options['language']].title()}")++ language: str = decode_options["language"]+ task: str = decode_options.get("task", "transcribe")+ tokenizer = get_tokenizer(+ model.is_multilingual,+ num_languages=model.num_languages,+ language=language,+ task=task,+ )++ if isinstance(clip_timestamps, str):+ clip_timestamps = [+ float(ts) for ts in (clip_timestamps.split(",") if clip_timestamps else [])+ ]+ seek_points: List[int] = [round(ts * FRAMES_PER_SECOND) for ts in clip_timestamps]+ if len(seek_points) == 0:+ seek_points.append(0)+ if len(seek_points) % 2 == 1:+ seek_points.append(content_frames)+ seek_clips: List[Tuple[int, int]] = list(zip(seek_points[::2], seek_points[1::2]))++ punctuation = "\"'“¿([{-\"'.。,,!!??::”)]}、"++ if word_timestamps and task == "translate":+ warnings.warn("Word-level timestamps on translations may not be reliable.")++ def decode_with_fallback(segment: torch.Tensor) -> DecodingResult:+ temperatures = [temperature] if isinstance(temperature, (int, float)) else temperature+ decode_result = None++ for t in temperatures:+ kwargs = {**decode_options}+ if t > 0:+ # disable beam_size and patience when t > 0+ kwargs.pop("beam_size", None)+ kwargs.pop("patience", None)+ else:+ # disable best_of when t == 0+ kwargs.pop("best_of", None)++ options = DecodingOptions(**kwargs, temperature=t)+ decode_result = model.decode(segment, options)++ needs_fallback = False+ if (+ compression_ratio_threshold is not None+ and decode_result.compression_ratio > compression_ratio_threshold+ ):+ needs_fallback = True # too repetitive+ if logprob_threshold is not None and decode_result.avg_logprob < logprob_threshold:+ needs_fallback = True # average log probability is too low+ if (+ no_speech_threshold is not None+ and decode_result.no_speech_prob > no_speech_threshold+ ):+ needs_fallback = False # silence+ if not needs_fallback:+ break++ return decode_result++ clip_idx = 0+ seek = seek_clips[clip_idx][0]+ input_stride = exact_div(N_FRAMES, model.dims.n_audio_ctx) # mel frames per output token: 2+ time_precision = (+ input_stride * HOP_LENGTH / SAMPLE_RATE+ ) # time per output token: 0.02 (seconds)+ all_tokens = []+ all_segments = []+ prompt_reset_since = 0++ if initial_prompt is not None:+ initial_prompt_tokens = tokenizer.encode(" " + initial_prompt.strip())+ all_tokens.extend(initial_prompt_tokens)+ else:+ initial_prompt_tokens = []++ def new_segment(*, start: float, end: float, tokens: torch.Tensor, result: DecodingResult):+ tokens = tokens.tolist()+ text_tokens = [token for token in tokens if token < tokenizer.eot]+ return {+ "seek": seek,+ "start": start,+ "end": end,+ "text": tokenizer.decode(text_tokens),+ "tokens": tokens,+ "temperature": result.temperature,+ "avg_logprob": result.avg_logprob,+ "compression_ratio": result.compression_ratio,+ "no_speech_prob": result.no_speech_prob,+ }++ # show the progress bar when verbose is False (if True, transcribed text will be printed)+ with tqdm.tqdm(total=content_frames, unit="frames", disable=verbose is not False) as pbar:+ last_speech_timestamp = 0.0+ # NOTE: This loop is obscurely flattened to make the diff readable.+ # A later commit should turn this into a simpler nested loop.+ # for seek_clip_start, seek_clip_end in seek_clips:+ # while seek < seek_clip_end+ while clip_idx < len(seek_clips):+ seek_clip_start, seek_clip_end = seek_clips[clip_idx]+ if seek < seek_clip_start:+ seek = seek_clip_start+ if seek >= seek_clip_end:+ clip_idx += 1+ if clip_idx < len(seek_clips):+ seek = seek_clips[clip_idx][0]+ continue+ time_offset = float(seek * HOP_LENGTH / SAMPLE_RATE)+ window_end_time = float((seek + N_FRAMES) * HOP_LENGTH / SAMPLE_RATE)+ segment_size = min(N_FRAMES, content_frames - seek, seek_clip_end - seek)+ mel_segment = mel[:, seek : seek + segment_size]+ segment_duration = segment_size * HOP_LENGTH / SAMPLE_RATE+ mel_segment = pad_or_trim(mel_segment, N_FRAMES).to(model.device).to(dtype)++ decode_options["prompt"] = all_tokens[prompt_reset_since:]+ result: DecodingResult = decode_with_fallback(mel_segment)+ tokens = torch.tensor(result.tokens)++ if no_speech_threshold is not None:+ # no voice activity check+ should_skip = result.no_speech_prob > no_speech_threshold+ if logprob_threshold is not None and result.avg_logprob > logprob_threshold:+ # don't skip if the logprob is high enough, despite the no_speech_prob+ should_skip = False++ if should_skip:+ seek += segment_size # fast-forward to the next segment boundary+ continue++ previous_seek = seek+ current_segments = []++ # anomalous words are very long/short/improbable+ def word_anomaly_score(word: dict) -> float:+ probability = word.get("probability", 0.0)+ duration = word["end"] - word["start"]+ score = 0.0+ if probability < 0.15:+ score += 1.0+ if duration < 0.133:+ score += (0.133 - duration) * 15+ if duration > 2.0:+ score += duration - 2.0+ return score++ def is_segment_anomaly(segment: Optional[dict]) -> bool:+ if segment is None or not segment["words"]:+ return False+ words = [w for w in segment["words"] if w["word"] not in punctuation]+ words = words[:8]+ score = sum(word_anomaly_score(w) for w in words)+ return score >= 3 or score + 0.01 >= len(words)++ def next_words_segment(segments: List[dict]) -> Optional[dict]:+ return next((s for s in segments if s["words"]), None)++ timestamp_tokens: torch.Tensor = tokens.ge(tokenizer.timestamp_begin)+ single_timestamp_ending = timestamp_tokens[-2:].tolist() == [False, True]++ consecutive = torch.where(timestamp_tokens[:-1] & timestamp_tokens[1:])[0]+ consecutive.add_(1)+ if len(consecutive) > 0:+ # if the output contains two consecutive timestamp tokens+ slices = consecutive.tolist()+ if single_timestamp_ending:+ slices.append(len(tokens))++ last_slice = 0+ for current_slice in slices:+ sliced_tokens = tokens[last_slice:current_slice]+ start_timestamp_pos = sliced_tokens[0].item() - tokenizer.timestamp_begin+ end_timestamp_pos = sliced_tokens[-1].item() - tokenizer.timestamp_begin+ current_segments.append(+ new_segment(+ start=time_offset + start_timestamp_pos * time_precision,+ end=time_offset + end_timestamp_pos * time_precision,+ tokens=sliced_tokens,+ result=result,+ )+ )+ last_slice = current_slice++ if single_timestamp_ending:+ # single timestamp at the end means no speech after the last timestamp.+ seek += segment_size+ else:+ # otherwise, ignore the unfinished segment and seek to the last timestamp+ last_timestamp_pos = tokens[last_slice - 1].item() - tokenizer.timestamp_begin+ seek += last_timestamp_pos * input_stride+ else:+ duration = segment_duration+ timestamps = tokens[timestamp_tokens.nonzero().flatten()]+ if len(timestamps) > 0 and timestamps[-1].item() != tokenizer.timestamp_begin:+ # no consecutive timestamps but it has a timestamp; use the last one.+ last_timestamp_pos = timestamps[-1].item() - tokenizer.timestamp_begin+ duration = last_timestamp_pos * time_precision++ current_segments.append(+ new_segment(+ start=time_offset,+ end=time_offset + duration,+ tokens=tokens,+ result=result,+ )+ )+ seek += segment_size++ if word_timestamps:+ add_word_timestamps(+ segments=current_segments,+ model=model,+ tokenizer=tokenizer,+ mel=mel_segment,+ num_frames=segment_size,+ prepend_punctuations=prepend_punctuations,+ append_punctuations=append_punctuations,+ last_speech_timestamp=last_speech_timestamp,+ )++ if not single_timestamp_ending:+ last_word_end = get_end(current_segments)+ if last_word_end is not None and last_word_end > time_offset:+ seek = round(last_word_end * FRAMES_PER_SECOND)++ # skip silence before possible hallucinations+ if hallucination_silence_threshold is not None:+ threshold = hallucination_silence_threshold+ if not single_timestamp_ending:+ last_word_end = get_end(current_segments)+ if last_word_end is not None and last_word_end > time_offset:+ remaining_duration = window_end_time - last_word_end+ if remaining_duration > threshold:+ seek = round(last_word_end * FRAMES_PER_SECOND)+ else:+ seek = previous_seek + segment_size++ # if first segment might be a hallucination, skip leading silence+ first_segment = next_words_segment(current_segments)+ if first_segment is not None and is_segment_anomaly(first_segment):+ gap = first_segment["start"] - time_offset+ if gap > threshold:+ seek = previous_seek + round(gap * FRAMES_PER_SECOND)+ continue++ # skip silence before any possible hallucination that is surrounded+ # by silence or more hallucinations+ hal_last_end = last_speech_timestamp+ for si in range(len(current_segments)):+ segment = current_segments[si]+ if not segment["words"]:+ continue+ if is_segment_anomaly(segment):+ next_segment = next_words_segment(current_segments[si + 1 :])+ if next_segment is not None:+ hal_next_start = next_segment["words"][0]["start"]+ else:+ hal_next_start = time_offset + segment_duration+ silence_before = (+ segment["start"] - hal_last_end > threshold+ or segment["start"] < threshold+ or segment["start"] - time_offset < 2.0+ )+ silence_after = (+ hal_next_start - segment["end"] > threshold+ or is_segment_anomaly(next_segment)+ or window_end_time - segment["end"] < 2.0+ )+ if silence_before and silence_after:+ seek = round(+ max(time_offset + 1, segment["start"]) * FRAMES_PER_SECOND+ )+ if content_duration - segment["end"] < threshold:+ seek = content_frames+ current_segments[si:] = []+ break+ hal_last_end = segment["end"]++ last_word_end = get_end(current_segments)+ if last_word_end is not None:+ last_speech_timestamp = last_word_end++ if verbose:+ for segment in current_segments:+ start, end, text = segment["start"], segment["end"], segment["text"]+ line = f"[{format_timestamp(start)} --> {format_timestamp(end)}] {text}"+ print(make_safe(line))++ # if a segment is instantaneous or does not contain text, clear it+ for i, segment in enumerate(current_segments):+ if segment["start"] == segment["end"] or segment["text"].strip() == "":+ segment["text"] = ""+ segment["tokens"] = []+ segment["words"] = []++ all_segments.extend(+ [+ {"id": i, **segment}+ for i, segment in enumerate(current_segments, start=len(all_segments))+ ]+ )+ all_tokens.extend(+ [token for segment in current_segments for token in segment["tokens"]]+ )++ if not condition_on_previous_text or result.temperature > 0.5:+ # do not feed the prompt tokens if a high temperature was used+ prompt_reset_since = len(all_tokens)++ # update progress bar+ pbar.update(min(content_frames, seek) - previous_seek)++ return dict(+ text=tokenizer.decode(all_tokens[len(initial_prompt_tokens) :]),+ segments=all_segments,+ language=language,+ )+++def cli():+ from . import available_models++ def valid_model_name(name):+ if name in available_models() or os.path.exists(name):+ return name+ raise ValueError(+ f"model should be one of {available_models()} or path to a model checkpoint"+ )++ # fmt: off+ parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)+ parser.add_argument("audio", nargs="+", type=str, help="audio file(s) to transcribe")+ parser.add_argument("--model", default="small", type=valid_model_name, help="name of the Whisper model to use")+ parser.add_argument("--model_dir", type=str, default=None, help="the path to save model files; uses ~/.cache/whisper by default")+ parser.add_argument("--device", default="cuda" if torch.cuda.is_available() else "cpu", help="device to use for PyTorch inference")+ parser.add_argument("--output_dir", "-o", type=str, default=".", help="directory to save the outputs")+ parser.add_argument("--output_format", "-f", type=str, default="all", choices=["txt", "vtt", "srt", "tsv", "json", "all"], help="format of the output file; if not specified, all available formats will be produced")+ parser.add_argument("--verbose", type=str2bool, default=True, help="whether to print out the progress and debug messages")++ parser.add_argument("--task", type=str, default="transcribe", choices=["transcribe", "translate"], help="whether to perform X->X speech recognition ('transcribe') or X->English translation ('translate')")+ parser.add_argument("--language", type=str, default=None, choices=sorted(LANGUAGES.keys()) + sorted([k.title() for k in TO_LANGUAGE_CODE.keys()]), help="language spoken in the audio, specify None to perform language detection")++ parser.add_argument("--temperature", type=float, default=0, help="temperature to use for sampling")+ parser.add_argument("--best_of", type=optional_int, default=5, help="number of candidates when sampling with non-zero temperature")+ parser.add_argument("--beam_size", type=optional_int, default=5, help="number of beams in beam search, only applicable when temperature is zero")+ parser.add_argument("--patience", type=float, default=None, help="optional patience value to use in beam decoding, as in https://arxiv.org/abs/2204.05424, the default (1.0) is equivalent to conventional beam search")+ parser.add_argument("--length_penalty", type=float, default=None, help="optional token length penalty coefficient (alpha) as in https://arxiv.org/abs/1609.08144, uses simple length normalization by default")++ parser.add_argument("--suppress_tokens", type=str, default="-1", help="comma-separated list of token ids to suppress during sampling; '-1' will suppress most special characters except common punctuations")+ parser.add_argument("--initial_prompt", type=str, default=None, help="optional text to provide as a prompt for the first window.")+ parser.add_argument("--condition_on_previous_text", type=str2bool, default=True, help="if True, provide the previous output of the model as a prompt for the next window; disabling may make the text inconsistent across windows, but the model becomes less prone to getting stuck in a failure loop")+ parser.add_argument("--fp16", type=str2bool, default=True, help="whether to perform inference in fp16; True by default")++ parser.add_argument("--temperature_increment_on_fallback", type=optional_float, default=0.2, help="temperature to increase when falling back when the decoding fails to meet either of the thresholds below")+ parser.add_argument("--compression_ratio_threshold", type=optional_float, default=2.4, help="if the gzip compression ratio is higher than this value, treat the decoding as failed")+ parser.add_argument("--logprob_threshold", type=optional_float, default=-1.0, help="if the average log probability is lower than this value, treat the decoding as failed")+ parser.add_argument("--no_speech_threshold", type=optional_float, default=0.6, help="if the probability of the <|nospeech|> token is higher than this value AND the decoding has failed due to `logprob_threshold`, consider the segment as silence")+ parser.add_argument("--word_timestamps", type=str2bool, default=False, help="(experimental) extract word-level timestamps and refine the results based on them")+ parser.add_argument("--prepend_punctuations", type=str, default="\"\'“¿([{-", help="if word_timestamps is True, merge these punctuation symbols with the next word")+ parser.add_argument("--append_punctuations", type=str, default="\"\'.。,,!!??::”)]}、", help="if word_timestamps is True, merge these punctuation symbols with the previous word")+ parser.add_argument("--highlight_words", type=str2bool, default=False, help="(requires --word_timestamps True) underline each word as it is spoken in srt and vtt")+ parser.add_argument("--max_line_width", type=optional_int, default=None, help="(requires --word_timestamps True) the maximum number of characters in a line before breaking the line")+ parser.add_argument("--max_line_count", type=optional_int, default=None, help="(requires --word_timestamps True) the maximum number of lines in a segment")+ parser.add_argument("--max_words_per_line", type=optional_int, default=None, help="(requires --word_timestamps True, no effect with --max_line_width) the maximum number of words in a segment")+ parser.add_argument("--threads", type=optional_int, default=0, help="number of threads used by torch for CPU inference; supercedes MKL_NUM_THREADS/OMP_NUM_THREADS")+ parser.add_argument("--clip_timestamps", type=str, default="0", help="comma-separated list start,end,start,end,... timestamps (in seconds) of clips to process, where the last end timestamp defaults to the end of the file")+ parser.add_argument("--hallucination_silence_threshold", type=optional_float, help="(requires --word_timestamps True) skip silent periods longer than this threshold (in seconds) when a possible hallucination is detected")+ # fmt: on++ args = parser.parse_args().__dict__+ model_name: str = args.pop("model")+ model_dir: str = args.pop("model_dir")+ output_dir: str = args.pop("output_dir")+ output_format: str = args.pop("output_format")+ device: str = args.pop("device")+ os.makedirs(output_dir, exist_ok=True)++ if model_name.endswith(".en") and args["language"] not in {"en", "English"}:+ if args["language"] is not None:+ warnings.warn(+ f"{model_name} is an English-only model but receipted '{args['language']}'; using English instead."+ )+ args["language"] = "en"++ temperature = args.pop("temperature")+ if (increment := args.pop("temperature_increment_on_fallback")) is not None:+ temperature = tuple(np.arange(temperature, 1.0 + 1e-6, increment))+ else:+ temperature = [temperature]++ if (threads := args.pop("threads")) > 0:+ torch.set_num_threads(threads)++ from . import load_model++ model = load_model(model_name, device=device, download_root=model_dir)++ writer = get_writer(output_format, output_dir)+ word_options = [+ "highlight_words",+ "max_line_count",+ "max_line_width",+ "max_words_per_line",+ ]+ if not args["word_timestamps"]:+ for option in word_options:+ if args[option]:+ parser.error(f"--{option} requires --word_timestamps True")+ if args["max_line_count"] and not args["max_line_width"]:+ warnings.warn("--max_line_count has no effect without --max_line_width")+ if args["max_words_per_line"] and args["max_line_width"]:+ warnings.warn("--max_words_per_line has no effect with --max_line_width")+ writer_args = {arg: args.pop(arg) for arg in word_options}+ for audio_path in args.pop("audio"):+ try:+ result = transcribe(model, audio_path, temperature=temperature, **args)+ writer(result, audio_path, **writer_args)+ except Exception as e:+ traceback.print_exc()+ print(f"Skipping {audio_path} due to {type(e).__name__}: {str(e)}")+++if __name__ == "__main__":+ cli()diff --git a/funasr/models/sense_voice/whisper_lib/triton_ops.py b/funasr/models/sense_voice/whisper_lib/triton_ops.pynew file mode 100644index 000000000..9919595ce--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/triton_ops.py@@ -0,0 +1,102 @@+from functools import lru_cache++import numpy as np+import torch++try:+ import triton+ import triton.language as tl+except ImportError:+ raise RuntimeError("triton import failed; try `pip install --pre triton`")+++def dtw_kernel(cost, trace, x, x_stride, cost_stride, trace_stride, N, M, BLOCK_SIZE: tl.constexpr):+ offsets = tl.arange(0, BLOCK_SIZE)+ mask = offsets < M++ for k in range(1, N + M + 1): # k = i + j+ tl.debug_barrier()++ p0 = cost + (k - 1) * cost_stride+ p1 = cost + k * cost_stride+ p2 = cost + k * cost_stride + 1++ c0 = tl.load(p0 + offsets, mask=mask)+ c1 = tl.load(p1 + offsets, mask=mask)+ c2 = tl.load(p2 + offsets, mask=mask)++ x_row = tl.load(x + (k - 1) * x_stride + offsets, mask=mask, other=0)+ cost_row = x_row + tl.minimum(tl.minimum(c0, c1), c2)++ cost_ptr = cost + (k + 1) * cost_stride + 1+ tl.store(cost_ptr + offsets, cost_row, mask=mask)++ trace_ptr = trace + (k + 1) * trace_stride + 1+ tl.store(trace_ptr + offsets, 2, mask=mask & (c2 <= c0) & (c2 <= c1))+ tl.store(trace_ptr + offsets, 1, mask=mask & (c1 <= c0) & (c1 <= c2))+ tl.store(trace_ptr + offsets, 0, mask=mask & (c0 <= c1) & (c0 <= c2))+++@lru_cache(maxsize=None)+def median_kernel(filter_width: int):+ @triton.jit+ def kernel(y, x, x_stride, y_stride, BLOCK_SIZE: tl.constexpr): # x.shape[-1] == filter_width+ row_idx = tl.program_id(0)+ offsets = tl.arange(0, BLOCK_SIZE)+ mask = offsets < y_stride++ x_ptr = x + row_idx * x_stride # noqa: F841+ y_ptr = y + row_idx * y_stride++ LOAD_ALL_ROWS_HERE # noqa: F821++ BUBBLESORT_HERE # noqa: F821++ tl.store(y_ptr + offsets, MIDDLE_ROW_HERE, mask=mask) # noqa: F821++ kernel = triton.JITFunction(kernel.fn)+ kernel.src = kernel.src.replace(+ " LOAD_ALL_ROWS_HERE",+ "\n".join(+ [f" row{i} = tl.load(x_ptr + offsets + {i}, mask=mask)" for i in range(filter_width)]+ ),+ )+ kernel.src = kernel.src.replace(+ " BUBBLESORT_HERE",+ "\n\n".join(+ [+ "\n\n".join(+ [+ "\n".join(+ [+ f" smaller = tl.where(row{j} < row{j + 1}, row{j}, row{j + 1})",+ f" larger = tl.where(row{j} > row{j + 1}, row{j}, row{j + 1})",+ f" row{j} = smaller",+ f" row{j + 1} = larger",+ ]+ )+ for j in range(filter_width - i - 1)+ ]+ )+ for i in range(filter_width // 2 + 1)+ ]+ ),+ )+ kernel.src = kernel.src.replace("MIDDLE_ROW_HERE", f"row{filter_width // 2}")++ return kernel+++def median_filter_cuda(x: torch.Tensor, filter_width: int):+ """Apply a median filter of given width along the last dimension of x"""+ slices = x.contiguous().unfold(-1, filter_width, 1)+ grid = np.prod(slices.shape[:-2])++ kernel = median_kernel(filter_width)+ y = torch.empty_like(slices[..., 0])++ BLOCK_SIZE = 1 << (y.stride(-2) - 1).bit_length()+ kernel[(grid,)](y, x, x.stride(-2), y.stride(-2), BLOCK_SIZE=BLOCK_SIZE)++ return ydiff --git a/funasr/models/sense_voice/whisper_lib/utils.py b/funasr/models/sense_voice/whisper_lib/utils.pynew file mode 100644index 000000000..5fc61256f--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/utils.py@@ -0,0 +1,283 @@+import json+import os+import re+import sys+import zlib+from typing import Callable, List, Optional, TextIO++system_encoding = sys.getdefaultencoding()++if system_encoding != "utf-8":++ def make_safe(string):+ # replaces any character not representable using the system default encoding with an '?',+ # avoiding UnicodeEncodeError (https://github.com/openai/whisper/discussions/729).+ return string.encode(system_encoding, errors="replace").decode(system_encoding)++else:++ def make_safe(string):+ # utf-8 can encode any Unicode code point, so no need to do the round-trip encoding+ return string+++def exact_div(x, y):+ assert x % y == 0+ return x // y+++def str2bool(string):+ str2val = {"True": True, "False": False}+ if string in str2val:+ return str2val[string]+ else:+ raise ValueError(f"Expected one of {set(str2val.keys())}, got {string}")+++def optional_int(string):+ return None if string == "None" else int(string)+++def optional_float(string):+ return None if string == "None" else float(string)+++def compression_ratio(text) -> float:+ text_bytes = text.encode("utf-8")+ return len(text_bytes) / len(zlib.compress(text_bytes))+++def format_timestamp(seconds: float, always_include_hours: bool = False, decimal_marker: str = "."):+ assert seconds >= 0, "non-negative timestamp expected"+ milliseconds = round(seconds * 1000.0)++ hours = milliseconds // 3_600_000+ milliseconds -= hours * 3_600_000++ minutes = milliseconds // 60_000+ milliseconds -= minutes * 60_000++ seconds = milliseconds // 1_000+ milliseconds -= seconds * 1_000++ hours_marker = f"{hours:02d}:" if always_include_hours or hours > 0 else ""+ return f"{hours_marker}{minutes:02d}:{seconds:02d}{decimal_marker}{milliseconds:03d}"+++def get_start(segments: List[dict]) -> Optional[float]:+ return next(+ (w["start"] for s in segments for w in s["words"]),+ segments[0]["start"] if segments else None,+ )+++def get_end(segments: List[dict]) -> Optional[float]:+ return next(+ (w["end"] for s in reversed(segments) for w in reversed(s["words"])),+ segments[-1]["end"] if segments else None,+ )+++class ResultWriter:+ extension: str++ def __init__(self, output_dir: str):+ self.output_dir = output_dir++ def __call__(self, result: dict, audio_path: str, options: Optional[dict] = None, **kwargs):+ audio_basename = os.path.basename(audio_path)+ audio_basename = os.path.splitext(audio_basename)[0]+ output_path = os.path.join(self.output_dir, audio_basename + "." + self.extension)++ with open(output_path, "w", encoding="utf-8") as f:+ self.write_result(result, file=f, options=options, **kwargs)++ def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ raise NotImplementedError+++class WriteTXT(ResultWriter):+ extension: str = "txt"++ def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ for segment in result["segments"]:+ print(segment["text"].strip(), file=file, flush=True)+++class SubtitlesWriter(ResultWriter):+ always_include_hours: bool+ decimal_marker: str++ def iterate_result(+ self,+ result: dict,+ options: Optional[dict] = None,+ *,+ max_line_width: Optional[int] = None,+ max_line_count: Optional[int] = None,+ highlight_words: bool = False,+ max_words_per_line: Optional[int] = None,+ ):+ options = options or {}+ max_line_width = max_line_width or options.get("max_line_width")+ max_line_count = max_line_count or options.get("max_line_count")+ highlight_words = highlight_words or options.get("highlight_words", False)+ max_words_per_line = max_words_per_line or options.get("max_words_per_line")+ preserve_segments = max_line_count is None or max_line_width is None+ max_line_width = max_line_width or 1000+ max_words_per_line = max_words_per_line or 1000++ def iterate_subtitles():+ line_len = 0+ line_count = 1+ # the next subtitle to yield (a list of word timings with whitespace)+ subtitle: List[dict] = []+ last: float = get_start(result["segments"]) or 0.0+ for segment in result["segments"]:+ chunk_index = 0+ words_count = max_words_per_line+ while chunk_index < len(segment["words"]):+ remaining_words = len(segment["words"]) - chunk_index+ if max_words_per_line > len(segment["words"]) - chunk_index:+ words_count = remaining_words+ for i, original_timing in enumerate(+ segment["words"][chunk_index : chunk_index + words_count]+ ):+ timing = original_timing.copy()+ long_pause = not preserve_segments and timing["start"] - last > 3.0+ has_room = line_len + len(timing["word"]) <= max_line_width+ seg_break = i == 0 and len(subtitle) > 0 and preserve_segments+ if line_len > 0 and has_room and not long_pause and not seg_break:+ # line continuation+ line_len += len(timing["word"])+ else:+ # new line+ timing["word"] = timing["word"].strip()+ if (+ len(subtitle) > 0+ and max_line_count is not None+ and (long_pause or line_count >= max_line_count)+ or seg_break+ ):+ # subtitle break+ yield subtitle+ subtitle = []+ line_count = 1+ elif line_len > 0:+ # line break+ line_count += 1+ timing["word"] = "\n" + timing["word"]+ line_len = len(timing["word"].strip())+ subtitle.append(timing)+ last = timing["start"]+ chunk_index += max_words_per_line+ if len(subtitle) > 0:+ yield subtitle++ if len(result["segments"]) > 0 and "words" in result["segments"][0]:+ for subtitle in iterate_subtitles():+ subtitle_start = self.format_timestamp(subtitle[0]["start"])+ subtitle_end = self.format_timestamp(subtitle[-1]["end"])+ subtitle_text = "".join([word["word"] for word in subtitle])+ if highlight_words:+ last = subtitle_start+ all_words = [timing["word"] for timing in subtitle]+ for i, this_word in enumerate(subtitle):+ start = self.format_timestamp(this_word["start"])+ end = self.format_timestamp(this_word["end"])+ if last != start:+ yield last, start, subtitle_text++ yield start, end, "".join(+ [+ re.sub(r"^(\s*)(.*)$", r"\1<u>\2</u>", word) if j == i else word+ for j, word in enumerate(all_words)+ ]+ )+ last = end+ else:+ yield subtitle_start, subtitle_end, subtitle_text+ else:+ for segment in result["segments"]:+ segment_start = self.format_timestamp(segment["start"])+ segment_end = self.format_timestamp(segment["end"])+ segment_text = segment["text"].strip().replace("-->", "->")+ yield segment_start, segment_end, segment_text++ def format_timestamp(self, seconds: float):+ return format_timestamp(+ seconds=seconds,+ always_include_hours=self.always_include_hours,+ decimal_marker=self.decimal_marker,+ )+++class WriteVTT(SubtitlesWriter):+ extension: str = "vtt"+ always_include_hours: bool = False+ decimal_marker: str = "."++ def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ print("WEBVTT\n", file=file)+ for start, end, text in self.iterate_result(result, options, **kwargs):+ print(f"{start} --> {end}\n{text}\n", file=file, flush=True)+++class WriteSRT(SubtitlesWriter):+ extension: str = "srt"+ always_include_hours: bool = True+ decimal_marker: str = ","++ def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ for i, (start, end, text) in enumerate(+ self.iterate_result(result, options, **kwargs), start=1+ ):+ print(f"{i}\n{start} --> {end}\n{text}\n", file=file, flush=True)+++class WriteTSV(ResultWriter):+ """+ Write a transcript to a file in TSV (tab-separated values) format containing lines like:+ <start time in integer milliseconds>\t<end time in integer milliseconds>\t<transcript text>++ Using integer milliseconds as start and end times means there's no chance of interference from+ an environment setting a language encoding that causes the decimal in a floating point number+ to appear as a comma; also is faster and more efficient to parse & store, e.g., in C++.+ """++ extension: str = "tsv"++ def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ print("start", "end", "text", sep="\t", file=file)+ for segment in result["segments"]:+ print(round(1000 * segment["start"]), file=file, end="\t")+ print(round(1000 * segment["end"]), file=file, end="\t")+ print(segment["text"].strip().replace("\t", " "), file=file, flush=True)+++class WriteJSON(ResultWriter):+ extension: str = "json"++ def write_result(self, result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ json.dump(result, file)+++def get_writer(output_format: str, output_dir: str) -> Callable[[dict, TextIO, dict], None]:+ writers = {+ "txt": WriteTXT,+ "vtt": WriteVTT,+ "srt": WriteSRT,+ "tsv": WriteTSV,+ "json": WriteJSON,+ }++ if output_format == "all":+ all_writers = [writer(output_dir) for writer in writers.values()]++ def write_all(result: dict, file: TextIO, options: Optional[dict] = None, **kwargs):+ for writer in all_writers:+ writer(result, file, options, **kwargs)++ return write_all++ return writers[output_format](output_dir)diff --git a/funasr/models/sense_voice/whisper_lib/version.py b/funasr/models/sense_voice/whisper_lib/version.pynew file mode 100644index 000000000..c96dd9ce4--- /dev/null+++ b/funasr/models/sense_voice/whisper_lib/version.py@@ -0,0 +1 @@+__version__ = "20231117"