09.RK3588 本地TTS语音合成
本文还是在python 虚拟环境里进行
source /userdata/aidemo/sensevoice/venv_sherpa_backup/bin/activate
1.RK3588里安装依赖
pip install soundfile
2.下载语音合成模型到板子上
wget https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-vits-zh-ll.tar.bz2
tar -xjf sherpa-onnx-vits-zh-ll.tar.bz2
rm sherpa-onnx-vits-zh-ll.tar.bz2
mv sherpa-onnx-vits-zh-ll tts
sherpa-onnx-zh-ll 介绍
一、基础定位
- 架构：VITS 端到端中文普通话 TTS 模型
- ll = long list，代表5 种内置音色（多说话人），无需 espeak-ng 音标库，纯中文专用
- 适配：sherpa-onnx 全平台（x86/ARM/RK3588 嵌入式、Android/Linux）
- 无需 --vits-data-dir，仅靠 lexicon.txt + tokens.txt + model.onnx 即可运行，完美匹配你当前目录结构
二、核心硬件 / 音频参数
- 采样率 24000Hz比 16kHz 模型人声更饱满、细节多，听感优于 aishell3 16k 模型
- 模型包总大小约 115MB，onnx 主模型浮点版，支持 RKNN 量化后跑 RK3588 NPU
- 内置5 个说话人 sid（0~4）
- sid=0：默认温柔女声
- sid=1/3/4：男声
- sid=2：清亮女声
- 原生内置数字、日期、多音字文本归一化逻辑，搭配你目录的number.fst/date.fst可完美朗读阿拉伯数字
3.编写测试脚本
tts.py如下
#!/usr/bin/env python3
# RK3588 TTS 修复：脚本自动出声、可调语速、数字正常朗读
import argparse
import logging
import sys
import time
import subprocess
import os
import numpy as np
import sherpa_onnx
import soundfile as sf
def add_vits_args(parser):
parser.add_argument(
"--vits-model",
type=str,
default="./model.onnx",
help="Path to vits model.onnx",
)
parser.add_argument(
"--vits-lexicon",
type=str,
default="./lexicon.txt",
help="Path to lexicon.txt",
)
parser.add_argument(
"--vits-tokens",
type=str,
default="./tokens.txt",
help="Path to tokens.txt",
)
parser.add_argument(
"--vits-data-dir",
type=str,
default="",
help="espeak-ng dict directory，当前模型留空",
)
def add_matcha_args(parser):
parser.add_argument("--matcha-acoustic-model", type=str, default="")
parser.add_argument("--matcha-vocoder", type=str, default="")
parser.add_argument("--matcha-lexicon", type=str, default="")
parser.add_argument("--matcha-tokens", type=str, default="")
parser.add_argument("--matcha-data-dir", type=str, default="")
def add_kokoro_args(parser):
parser.add_argument("--kokoro-model", type=str, default="")
parser.add_argument("--kokoro-voices", type=str, default="")
parser.add_argument("--kokoro-tokens", type=str, default="")
parser.add_argument("--kokoro-data-dir", type=str, default="")
parser.add_argument("--kokoro-lexicon", type=str, default="")
def add_kitten_args(parser):
parser.add_argument("--kitten-model", type=str, default="")
parser.add_argument("--kitten-voices", type=str, default="")
parser.add_argument("--kitten-tokens", type=str, default="")
parser.add_argument("--kitten-data-dir", type=str, default="")
def get_args():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="RK3588 offline TTS fix speed fast & number silent"
)
add_vits_args(parser)
add_matcha_args(parser)
add_kokoro_args(parser)
add_kitten_args(parser)
# 关键：加载数字、日期转换规则文件，多个fst用逗号分隔
parser.add_argument(
"--tts-rule-fsts",
type=str,
default="./number.fst,./date.fst,./new_heteronym.fst,./phone.fst",
help="文本归一化规则，处理数字、日期、多音字",
)
parser.add_argument("--output-filename", type=str, default="./generated.wav")
parser.add_argument("--sid", type=int, default=0)
parser.add_argument("--debug", action="store_true")
parser.add_argument("--provider", type=str, default="rknpu", choices=["cpu", "rknpu"])
parser.add_argument("--num-threads", type=int, default=6)
# 默认语速0.9，放慢朗读速度
parser.add_argument("--speed", type=float, default=1, help="语速 <1慢 >1快")
parser.add_argument("--silence-scale", type=float, default=0.3, help="增大停顿，朗读更清晰")
parser.add_argument("--hw-card", type=int, default=0)
parser.add_argument("text", type=str)
return parser.parse_args()
def play_wav(wav_path: str):
# 清理残留音频进程，释放声卡
subprocess.run(["killall", "-q", "play", "aplay", "sox"], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
time.sleep(0.1)
cmd = ["play", wav_path, "-c", "2", "-q"]
logging.info(f"播放音频: {wav_path}")
ret = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if ret.returncode != 0:
logging.warning("立体声转换失败，原生播放")
subprocess.run(["play", wav_path])
# 缓冲等待，避免短音频截断无声
time.sleep(0.5)
logging.info("播放完成")
def main():
args = get_args()
log_level = logging.DEBUG if args.debug else logging.INFO
logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=log_level)
model_cfg = sherpa_onnx.OfflineTtsModelConfig(
vits=sherpa_onnx.OfflineTtsVitsModelConfig(
model=args.vits_model,
lexicon=args.vits_lexicon,
tokens=args.vits_tokens,
data_dir=args.vits_data_dir,
),
matcha=sherpa_onnx.OfflineTtsMatchaModelConfig(
acoustic_model=args.matcha_acoustic_model,
vocoder=args.matcha_vocoder,
lexicon=args.matcha_lexicon,
tokens=args.matcha_tokens,
data_dir=args.matcha_data_dir,
),
kokoro=sherpa_onnx.OfflineTtsKokoroModelConfig(
model=args.kokoro_model,
voices=args.kokoro_voices,
tokens=args.kokoro_tokens,
data_dir=args.kokoro_data_dir,
lexicon=args.kokoro_lexicon,
),
kitten=sherpa_onnx.OfflineTtsKittenModelConfig(
model=args.kitten_model,
voices=args.kitten_voices,
tokens=args.kitten_tokens,
data_dir=args.kitten_data_dir,
),
provider=args.provider,
num_threads=args.num_threads,
debug=args.debug,
)
tts_config = sherpa_onnx.OfflineTtsConfig(
model=model_cfg,
rule_fsts=args.tts_rule_fsts,
max_num_sentences=1
)
if not tts_config.validate():
raise ValueError("TTS配置错误，请检查模型与fst文件路径")
logging.info(f"加载TTS模型，后端={args.provider}，线程={args.num_threads}")
tts = sherpa_onnx.OfflineTts(tts_config)
sample_rate = tts.sample_rate
logging.info(f"模型加载完成，采样率 {sample_rate} Hz")
gen_cfg = sherpa_onnx.GenerationConfig()
gen_cfg.sid = args.sid
gen_cfg.speed = args.speed
gen_cfg.silence_scale = args.silence_scale
logging.info("开始合成语音...")
start = time.time()
audio = tts.generate(text=args.text, config=gen_cfg)
end = time.time()
if len(audio.samples) == 0:
logging.error("音频生成失败，为空")
return
sf.write(args.output_filename, audio.samples, samplerate=sample_rate, subtype="PCM_16")
dur = len(audio.samples) / sample_rate
cost = end - start
rtf = cost / dur
logging.info(f"待朗读文本: {args.text}")
logging.info(f"推理耗时 {cost:.3f}s | 音频时长 {dur:.3f}s | RTF {rtf:.3f}")
logging.info(f"音频保存路径: {os.path.abspath(args.output_filename)}")
play_wav(args.output_filename)
if __name__ == "__main__":
try:
main()
except KeyboardInterrupt:
print("\n检测Ctrl+C，程序退出")
subprocess.run(["killall", "-q", "play", "aplay"])
sys.exit(0)
此时文件结构如下
4.进行测试
python tts.py "你好,已经开启回家模式！灯已全开，空调26度制冷，新风系统已开启，沐浴热水系统开始预热"
执行结果如下
已经能正常合成语音
/project/sherpa-onnx/csrc/provider.cc:StringToProvider:35 Unsupported string: rknpu. Fallback to cpu
原因：之前pip install sherpa-onnx 从默认 pypi 下载的通用包，不带 RKNN/rknpu 后端，不支持 NPU 加速，自动降级 CPU 推理。
这个不影响功能，只是推理慢一点
可以试着卸载掉sherpa-onnx，安装带rknpu版本的
pip uninstall sherpa-onnx -y
pip install sherpa-onnx -f https://k2-fsa.github.io/sherpa/onnx/rk-npu-cn.html
4.边合成边播放
上述代码是合成完成后再播放，在合成完成前会有一段时间等待，用户体验不好，下文给出一边合成一边播放的方法
#!/usr/bin/env python3
# 中文VITS TTS 修复文字吞字无声、分片断层
import argparse
import logging
import queue
import sys
import threading
import time
import subprocess
import os
import tempfile
import numpy as np
import sherpa_onnx
import soundfile as sf
audio_queue = queue.Queue(maxsize=100)
started = False
stopped = False
killed = False
sample_rate = None
first_audio_time = None
first_play_time = None
model_loaded_time = None
# 文本归一化规则文件列表
FST_LIST = [
"./number.fst",
"./date.fst",
"./new_heteronym.fst",
"./phone.fst"
]
def check_fst_files():
missing = []
for f in FST_LIST:
if not os.path.exists(f):
missing.append(f)
if missing:
raise FileNotFoundError(f"缺失文本处理规则文件：{','.join(missing)}")
def add_vits_args(parser):
parser.add_argument("--vits-model", type=str, default="./model.onnx")
parser.add_argument("--vits-lexicon", type=str, default="./lexicon.txt")
parser.add_argument("--vits-tokens", type=str, default="./tokens.txt")
parser.add_argument("--vits-data-dir", type=str, default="")
def add_matcha_args(parser):
parser.add_argument("--matcha-acoustic-model", type=str, default="")
parser.add_argument("--matcha-vocoder", type=str, default="")
parser.add_argument("--matcha-lexicon", type=str, default="")
parser.add_argument("--matcha-tokens", type=str, default="")
parser.add_argument("--matcha-data-dir", type=str, default="")
def add_kokoro_args(parser):
parser.add_argument("--kokoro-model", type=str, default="")
parser.add_argument("--kokoro-voices", type=str, default="")
parser.add_argument("--kokoro-tokens", type=str, default="")
parser.add_argument("--kokoro-data-dir", type=str, default="")
parser.add_argument("--kokoro-lexicon", type=str, default="")
def add_kitten_args(parser):
parser.add_argument("--kitten-model", type=str, default="")
parser.add_argument("--kitten-voices", type=str, default="")
parser.add_argument("--kitten-tokens", type=str, default="")
parser.add_argument("--kitten-data-dir", type=str, default="")
def get_args():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
description="RK3588 中文VITS TTS"
)
add_vits_args(parser)
add_matcha_args(parser)
add_kokoro_args(parser)
add_kitten_args(parser)
parser.add_argument("--tts-rule-fsts", type=str, default=",".join(FST_LIST))
parser.add_argument("--output-filename", type=str, default="./cn_out.wav")
parser.add_argument("--sid", type=int, default=0)
parser.add_argument("--debug", action="store_true")
parser.add_argument("--provider", type=str, default="rknpu", choices=["cpu", "rknpu"])
parser.add_argument("--num-threads", type=int, default=6)
parser.add_argument("--speed", type=float, default=1.0)
parser.add_argument("--silence-scale", type=float, default=0.4)
parser.add_argument("--hw-card", type=int, default=0)
parser.add_argument("text", type=str)
return parser.parse_args()
def generated_audio_callback(samples: np.ndarray, progress: float):
global first_audio_time, started
if killed:
return 0
if first_audio_time is None:
first_audio_time = time.time()
if audio_queue.full():
try:
dropped = audio_queue.get_nowait()
except queue.Empty:
pass
while audio_queue.qsize() >= 80:
time.sleep(0.05)
audio_queue.put(samples)
if not started:
started = True
return 1
def play_worker(card_id: int):
global killed, stopped, sample_rate, first_play_time
subprocess.run(["killall", "-q", "aplay", "sox", "play"],
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
time.sleep(0.1)
target_sample_rate = 48000
up_ratio = target_sample_rate // sample_rate
play_proc = subprocess.Popen(
["play", "-c", "2", "-r", str(target_sample_rate), "-e", "signed-integer",
"-b", "16", "-t", "raw", "-"],
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
while True:
if killed or (stopped and audio_queue.empty()):
break
try:
chunk = audio_queue.get(timeout=0.1)
except queue.Empty:
continue
try:
if chunk.dtype == np.float32 or chunk.dtype == np.float64:
chunk = (chunk * 32767).astype(np.int16)
chunk_up = np.repeat(chunk, up_ratio)
chunk_stereo = np.column_stack([chunk_up, chunk_up]).flatten()
if first_play_time is None:
first_play_time = time.time()
play_proc.stdin.write(chunk_stereo.tobytes())
except Exception as e:
logging.warning(f"管道写入失败 {e}")
break
play_proc.stdin.close()
play_proc.wait()
def main():
args = get_args()
log_level = logging.DEBUG if args.debug else logging.INFO
logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=log_level)
check_fst_files()
tts_model_cfg = sherpa_onnx.OfflineTtsModelConfig(
vits=sherpa_onnx.OfflineTtsVitsModelConfig(
model=args.vits_model,
lexicon=args.vits_lexicon,
tokens=args.vits_tokens,
data_dir=args.vits_data_dir,
),
matcha=sherpa_onnx.OfflineTtsMatchaModelConfig(),
kokoro=sherpa_onnx.OfflineTtsKokoroModelConfig(),
kitten=sherpa_onnx.OfflineTtsKittenModelConfig(),
provider=args.provider,
num_threads=args.num_threads,
debug=args.debug,
)
tts_cfg = sherpa_onnx.OfflineTtsConfig(
model=tts_model_cfg,
rule_fsts=args.tts_rule_fsts,
max_num_sentences=1
)
if not tts_cfg.validate():
raise ValueError("模型配置校验失败")
logging.info(f"加载中文VITS TTS | 后端:{args.provider} 线程:{args.num_threads}")
tts = sherpa_onnx.OfflineTts(tts_cfg)
global sample_rate
sample_rate = tts.sample_rate
global model_loaded_time
model_loaded_time = time.time()
logging.info(f"模型加载完成，采样率 {sample_rate} Hz")
play_th = threading.Thread(target=play_worker, args=(args.hw_card,), daemon=True)
play_th.start()
time.sleep(0.5)
gen_cfg = sherpa_onnx.GenerationConfig()
gen_cfg.sid = args.sid
gen_cfg.speed = args.speed
gen_cfg.silence_scale = args.silence_scale
logging.info("开始流式合成")
start_t = time.time()
audio_full = tts.generate(args.text, gen_cfg, callback=generated_audio_callback)
end_t = time.time()
global stopped
stopped = True
sf.write(args.output_filename, audio_full.samples, samplerate=sample_rate, subtype="PCM_16")
audio_dur = len(audio_full.samples) / sample_rate
cost = end_t - start_t
rtf = cost / audio_dur
tts_start_delay = start_t - model_loaded_time
first_audio_delay = first_audio_time - start_t if first_audio_time else -1
first_play_delay = first_play_time - start_t if first_play_time else -1
time_to_hear = first_play_time - model_loaded_time if first_play_time else -1
logging.info(f"输入文本：{args.text}")
logging.info(f"延迟统计：")
logging.info(f" - 模型加载完成到开始合成: {tts_start_delay:.3f} s")
logging.info(f" - 开始合到成第一帧音频: {first_audio_delay:.3f} s")
logging.info(f" - 开始合到成开始播放: {first_play_delay:.3f} s")
logging.info(f" - 模型加载完成到听到声音: {time_to_hear:.3f} s")
logging.info(f"总推理耗时：{cost:.3f} s | 音频时长：{audio_dur:.3f} s | RTF：{rtf:.3f}")
logging.info(f"WAV文件保存路径：{os.path.abspath(args.output_filename)}")
play_th.join()
if __name__ == "__main__":
logging.basicConfig(format="%(asctime)s %(levelname)s %(message)s", level=logging.INFO)
try:
main()
except FileNotFoundError as e:
logging.error(f"文件缺失错误：{e}")
sys.exit(1)
except KeyboardInterrupt:
print("\n检测 Ctrl+C，程序退出")
killed = True
主流程解释：
1.创建 play_worker 播放线程（守护线程）
2.流式合成阶段
tts.generate(text, config, callback=generated_audio_callback)
- sherpa_onnx 开始合成，每生成一段音频就调用 generated_audio_callback
- 回调函数将音频块放入 audio_queue 队列
- 同时 播放线程从队列取出音频块，通过管道实时播放
执行结果
浙公网安备 33010602011771号