VoiceCraft：8.5K+ Star

# 1. Clone the repository
git clone https://github.com/jasonppy/VoiceCraft.git
cd VoiceCraft

# 2. Build the Docker image
docker build --tag "voicecraft" .

# 3. Start the container (Linux)
./start-jupyter.sh
# Or on Windows:
# start-jupyter.bat

# 4. Access Jupyter — copy the URL from logs
docker logs jupyter | grep "127.0.0.1:8888"

# 5. Verify GPU access inside the container
docker exec -it jupyter nvidia-smi

# Create and activate the environment
conda create -n voicecraft python=3.9.16
conda activate voicecraft

# Install PyTorch with CUDA 11.7
pip install torch==2.0.1 torchaudio==2.0.2 --index-url https://download.pytorch.org/whl/cu117

# Install audiocraft (EnCodec dependency)
pip install -e git+https://github.com/facebookresearch/audiocraft.git@c5157b5bf14bf83449c17ea1eeb66c19fb4bc7f0#egg=audiocraft

# Install xformers for memory-efficient attention
pip install xformers==0.0.22

# Core dependencies
pip install tensorboard==2.16.2
pip install phonemizer==3.2.1
pip install datasets==2.16.0
pip install torchmetrics==0.11.1
pip install huggingface_hub==0.22.2

# System dependencies
apt-get install -y ffmpeg espeak-ng

# Install Montreal Forced Aligner (MFA) for text-audio alignment
conda install -c conda-forge montreal-forced-aligner=2.2.17 openfst=1.8.2 kaldi=5.5.1068

# Download MFA English models
mfa model download dictionary english_us_arpa
mfa model download acoustic english_us_arpa

# Jupyter kernel (optional)
conda install -n voicecraft ipykernel --no-deps --force-reinstall

# Additional system dependencies for Gradio
apt-get install -y espeak espeak-data libespeak1 libespeak-dev
apt-get install -y festival build-essential flac libasound2-dev libsndfile1-dev

# Install Gradio requirements
pip install -r gradio_requirements.txt

# Launch the Gradio server
python gradio_app.py

# Launch the Gradio app with default settings
python gradio_app.py --model-name "giga330M" --device "cuda"

# With custom model path
python gradio_app.py   --model-path "./pretrained_models/giga330M.pth"   --codec-model "encodec_16khz"   --share  # Create a public URL

# inference_tts.ipynb — Zero-shot TTS example
from voicecraft import VoiceCraft

# Load the 330M model (faster, good quality)
model = VoiceCraft.from_pretrained("pyp1/VoiceCraft", subfolder="giga330M")

# Provide 3-5 seconds of reference audio
reference_audio = "demo/pam.wav"  # Your reference clip
reference_text = "I found the amazing VoiceCraft model"

# Text to synthesize
target_text = "This is a test of zero shot voice cloning with VoiceCraft"

# Generate
output = model.tts(
    target_text=target_text,
    reference_audio=reference_audio,
    reference_text=reference_text,
    top_k=40,  # March 2025 update: top-k=40 improves quality
    temperature=1.0
)
output.save("output_tts.wav")

# TTS inference via CLI
python tts_demo.py   --audio_path "demo/pam.wav"   --target_transcript "This is the text to speak"   --model_name "giga330M"   --top_k 40   --temperature 1.0   --output_path "output.wav"

# Speech editing via CLI
python speech_editing_demo.py   --audio_path "demo/pam.wav"   --original_transcript "original text here"   --edited_transcript "edited text here"   --model_name "giga830M"   --output_path "edited_output.wav"

# Dockerfile.api — Production API wrapper
FROM voicecraft:latest

WORKDIR /app
COPY api.py ./
COPY requirements-api.txt ./

RUN pip install -r requirements-api.txt

EXPOSE 8000

CMD ["uvicorn", "api:app", "--host", "0.0.0.0", "--port", "8000"]

# api.py — FastAPI wrapper for VoiceCraft
from fastapi import FastAPI, UploadFile, File
from voicecraft import VoiceCraft
import torchaudio

app = FastAPI()
model = VoiceCraft.from_pretrained("pyp1/VoiceCraft", subfolder="giga330M")

@app.post("/tts")
async def tts(
    audio: UploadFile = File(...),
    reference_text: str = "",
    target_text: str = ""
):
    """Zero-shot TTS endpoint."""
    ref_audio, sr = torchaudio.load(audio.file)
    output = model.tts(
        target_text=target_text,
        reference_audio=ref_audio,
        reference_text=reference_text,
        top_k=40
    )
    return {"output": output.serialize()}

from huggingface_hub import hf_hub_download

# Download model weights
model_path = hf_hub_download(
    repo_id="pyp1/VoiceCraft",
    filename="giga330M.pth",
    subfolder="",
    local_dir="./pretrained_models"
)

# Also available via ModelScope (for China region)
from modelscope import snapshot_download
model_dir = snapshot_download('AI-ModelScope/VoiceCraft')

# Enable kvcache for 8GB GPU inference
output = model.tts(
    target_text=target_text,
    reference_audio=reference_audio,
    reference_text=reference_text,
    top_k=40,
    kvcache=True,  # Reduces VRAM usage by ~60%
    batch_size=1
)

# Recommended: top-k=40 for best quality
output = model.tts(
    target_text=target_text,
    reference_audio=reference_audio,
    reference_text=reference_text,
    top_k=40,
    temperature=1.0
)

# Prepare your dataset
conda activate voicecraft
cd ./data
python phonemize_encodec_encode_hf.py   --dataset_size xs   --download_to /path/to/downloads   --save_dir /path/to/processed   --encodec_model_path /path/to/encodec   --mega_batch_size 120   --batch_size 32   --max_len 30000

# Start fine-tuning
cd ../z_scripts
bash e830M_ft.sh  # Fine-tune 830M model

import logging
from torch.utils.tensorboard import SummaryWriter

# Setup logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger("voicecraft")

# TensorBoard for training monitoring
writer = SummaryWriter(log_dir="./runs/voicecraft-ft")
writer.add_scalar("loss/train", loss.item(), global_step)
writer.add_scalar("mos/validation", val_mos, global_step)