From 7d67b4af6c20ef22b62650fc032b242f2582e294 Mon Sep 17 00:00:00 2001 From: Dita Aji Pratama Date: Fri, 18 Sep 2026 17:02:51 +0700 Subject: [PATCH] TTS --- README.md | 23 ++++++++++++++- config/tts.py | 23 +++++++++++++++ core/tts_room.py | 7 +++++ tts_runner.py | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ usage-tts.sh | 19 ++++++++++++ usage.sh | 18 ++++++++++++ 6 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 config/tts.py create mode 100644 core/tts_room.py create mode 100644 tts_runner.py create mode 100755 usage-tts.sh create mode 100755 usage.sh diff --git a/README.md b/README.md index 7b3e0da..5063b56 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # STT Runner -Speech-to-Text transcription using sherpa-onnx + Qwen3-ASR. +Speech-to-Text transcription using sherpa-onnx + Qwen3-ASR, plus Text-to-Speech with ZipVoice (zero-shot voice cloning). ## Installation @@ -11,16 +11,27 @@ python3 -m venv .venv ## Usage +### Speech-to-Text + ```bash python stt_runner.py [--language=Indonesian] audio1.wav audio2.wav ... ``` +### Text-to-Speech + +```bash +python tts_runner.py [--output=out.wav] [--ref-audio=ref.wav] [--ref-text="..."] "text to speak" +``` + +Output defaults to `output.wav`. The reference audio/text (voice to clone) is set in `config/tts.py` and can be overridden per-run with `--ref-audio` / `--ref-text` (the text must match the audio exactly). + ## Configuration Model paths and inference parameters are hardcoded in `config/`: - `config/model.py` — model paths (conv_frontend, encoder, decoder, tokenizer under `models/`) - `config/asr.py` — inference params: `LANGUAGE`, `HOTWORDS`, `NUM_THREADS`, `PROVIDER`, `SAMPLE_RATE`, `FEATURE_DIM`, `MAX_TOTAL_LEN`, `MAX_NEW_TOKENS` +- `config/tts.py` — TTS model paths, `REFERENCE_AUDIO`, `REFERENCE_TEXT`, `OUTPUT_FILE`, `NUM_THREADS`, `PROVIDER`, `NUM_STEPS` `LANGUAGE` defaults to `""` (all languages / auto-detect). Passing `--language` on the CLI overrides it. @@ -35,4 +46,14 @@ wget -O models/model_1.7B/decoder.int8.onnx "$BASE/model_1.7B/decoder.int8.onnx for f in vocab.json merges.txt tokenizer_config.json preprocessor_config.json config.json chat_template.json; do wget -O "models/tokenizer/$f" "$BASE/tokenizer/$f" done +``` + +## Download Model (ZipVoice TTS) + +```bash +mkdir -p models/zipvoice +wget -qO- https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-zipvoice-distill-int8-zh-en-emilia.tar.bz2 \ + | tar xjf - -C models/zipvoice --strip-components=1 +wget -O models/zipvoice/vocos_24khz.onnx \ + https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx ``` \ No newline at end of file diff --git a/config/tts.py b/config/tts.py new file mode 100644 index 0000000..cd43b09 --- /dev/null +++ b/config/tts.py @@ -0,0 +1,23 @@ +from config.model import MODEL_DIR + +ZIPVOICE = MODEL_DIR / "zipvoice" + +TOKENS = ZIPVOICE / "tokens.txt" +ENCODER = ZIPVOICE / "encoder.int8.onnx" +DECODER = ZIPVOICE / "decoder.int8.onnx" +DATA_DIR = ZIPVOICE / "espeak-ng-data" +LEXICON = ZIPVOICE / "lexicon.txt" +VOCODER = ZIPVOICE / "vocos_24khz.onnx" + +# Reference voice for zero-shot voice cloning. +# REFERENCE_TEXT must match what is spoken in REFERENCE_AUDIO exactly, +# otherwise the cloned voice quality will noticeably degrade. +REFERENCE_AUDIO = ZIPVOICE / "test_wavs" / "leijun-1.wav" +REFERENCE_TEXT = "那还是三十六年前, 一九八七年. 我呢考上了武汉大学的计算机系." + +NUM_THREADS = 2 +PROVIDER = "cpu" +NUM_STEPS = 4 # Generation quality/speed tradeoff (higher = better, slower) + +# Default output filename. +OUTPUT_FILE = "output.wav" \ No newline at end of file diff --git a/core/tts_room.py b/core/tts_room.py new file mode 100644 index 0000000..76968df --- /dev/null +++ b/core/tts_room.py @@ -0,0 +1,7 @@ +import argparse + +parser = argparse.ArgumentParser() +parser.add_argument("--output", type=str, default=None, help="Output WAV file, overrides config OUTPUT_FILE") +parser.add_argument("--ref-audio", type=str, default=None, help="Reference audio for voice cloning, overrides config REFERENCE_AUDIO") +parser.add_argument("--ref-text", type=str, default=None, help="Reference text matching --ref-audio, overrides config REFERENCE_TEXT") +parser.add_argument("text", type=str, help="Text to synthesize into speech") \ No newline at end of file diff --git a/tts_runner.py b/tts_runner.py new file mode 100644 index 0000000..18f5993 --- /dev/null +++ b/tts_runner.py @@ -0,0 +1,75 @@ +import sys +import time +from pathlib import Path +import sherpa_onnx, soundfile as sf +from core import tts_room as ap +from config.tts import ( + TOKENS, ENCODER, DECODER, DATA_DIR, LEXICON, VOCODER, + REFERENCE_AUDIO, REFERENCE_TEXT, NUM_THREADS, PROVIDER, NUM_STEPS, OUTPUT_FILE, +) + +def tts_run(args): + + ref_audio = Path(args.ref_audio) if args.ref_audio is not None else Path(REFERENCE_AUDIO) + ref_text = args.ref_text if args.ref_text is not None else REFERENCE_TEXT + + if not ref_audio.is_file(): + print(f"Reference audio not found: {ref_audio}", file=sys.stderr) + return + + print("Loading model...") + tts_config = sherpa_onnx.OfflineTtsConfig( + model=sherpa_onnx.OfflineTtsModelConfig( + zipvoice=sherpa_onnx.OfflineTtsZipvoiceModelConfig( + tokens = str(TOKENS), + encoder = str(ENCODER), + decoder = str(DECODER), + data_dir = str(DATA_DIR), + lexicon = str(LEXICON), + vocoder = str(VOCODER), + ), + debug = False, + num_threads = NUM_THREADS, + provider = PROVIDER, + ) + ) + if not tts_config.validate(): + raise ValueError("Invalid TTS config. Please read the previous error messages.") + + tts = sherpa_onnx.OfflineTts(tts_config) + print("Model ready!") + + reference_audio, sr = sf.read(str(ref_audio), dtype="float32") + if reference_audio.ndim > 1: + reference_audio = reference_audio[:, 0] + + gen_config = sherpa_onnx.GenerationConfig() + gen_config.reference_audio = reference_audio + gen_config.reference_sample_rate = sr + gen_config.reference_text = ref_text + gen_config.num_steps = NUM_STEPS + + output_file = args.output if args.output is not None else OUTPUT_FILE + + print("Generating...") + start = time.time() + audio = tts.generate(args.text, gen_config) + elapsed = time.time() - start + + if len(audio.samples) == 0: + print("Error in generating audio. Please read the previous error messages.", file=sys.stderr) + return + + sf.write(output_file, audio.samples, samplerate=audio.sample_rate, subtype="PCM_16") + + duration = len(audio.samples) / audio.sample_rate + print() + print(f"Output : {output_file}") + print(f"Text : {args.text}") + print(f"Duration : {duration:.3f} s") + print(f"Elapsed : {elapsed:.3f} s") + print(f"RTF : {elapsed:.3f} / {duration:.3f} = {elapsed / duration:.3f}") + print() + +if __name__ == "__main__": + tts_run( ap.parser.parse_args() ) \ No newline at end of file diff --git a/usage-tts.sh b/usage-tts.sh new file mode 100755 index 0000000..1ffdbe2 --- /dev/null +++ b/usage-tts.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PYTHON="${PYTHON:-$BASE_DIR/.venv/bin/python}" + +if [ ! -x "$PYTHON" ]; then + echo "Venv not found. Create it with: python3 -m venv .venv && .venv/bin/pip install -r requirements.txt" >&2 + exit 1 +fi + +if [ $# -lt 1 ]; then + echo "Usage: $0 [--output ] [--ref-audio ] [--ref-text ]" >&2 + echo " Synthesize speech with ZipVoice zero-shot TTS (sherpa-onnx)." >&2 + echo " Reference audio/text defaults are set in config/tts.py." >&2 + exit 1 +fi + +"$PYTHON" "$BASE_DIR/tts_runner.py" "$@" diff --git a/usage.sh b/usage.sh new file mode 100755 index 0000000..6601711 --- /dev/null +++ b/usage.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PYTHON="${PYTHON:-$BASE_DIR/.venv/bin/python}" + +if [ ! -x "$PYTHON" ]; then + echo "Venv not found. Create it with: python3 -m venv .venv && .venv/bin/pip install -r requirements.txt" >&2 + exit 1 +fi + +if [ $# -lt 1 ]; then + echo "Usage: $0 " >&2 + echo " Transcribe audio with Qwen3-ASR 1.7B (sherpa-onnx)." >&2 + exit 1 +fi + +"$PYTHON" "$BASE_DIR/stt_runner.py" "$@"