Open-Sora：29K+ Stars 开源视频生成框架

# 核心推理管道（简化）
import torch
from opensora.models import STDiT3, T5Encoder, DC_AE
from opensora.schedulers import RectifiedFlowScheduler

# 加载模型
stdit = STDiT3.from_pretrained("hpcai-tech/Open-Sora-v2").cuda()
text_encoder = T5Encoder.from_pretrained("google/t5-v1_1-xxl").cuda()
vae = DC_AE.from_pretrained("hpcai-tech/Open-Sora-v2/vae").cuda()

# 编码提示
prompt = "宁静海底场景含海龟"
prompt_embed = text_encoder.encode(prompt)  # [1, 512, 4096]

# 初始化潜噪声
latent = torch.randn(1, 16, 16, 128, 128).cuda()  # [B, C, T, H, W]

# rectified flow 去噪
scheduler = RectifiedFlowScheduler(num_steps=50)
for t in scheduler.timesteps:
    noise_pred = stdit(latent, t, prompt_embed)
    latent = scheduler.step(noise_pred, t, latent)

# 解码到视频
video = vae.decode(latent)  # [1, 3, 65, 768, 768]

# 创建虚拟环境
conda create -n opensora python=3.10 -y
conda activate opensora

# 克隆仓库
git clone https://github.com/hpcaitech/Open-Sora.git
cd Open-Sora

# 安装 PyTorch（CUDA 12.1）
pip install torch==2.4.0 torchvision==0.19.0 --index-url https://download.pytorch.org/whl/cu121

# 安装 Open-Sora
pip install -v .

# 安装可选加速器
pip install xformers==0.0.27.post2 --index-url https://download.pytorch.org/whl/cu121
pip install flash-attn --no-build-isolation

# 克隆仓库
git clone https://github.com/hpcaitech/Open-Sora.git
cd Open-Sora

# 构建 Docker 镜像
docker build -t opensora:latest .

# 运行交互容器
docker run -ti --gpus all \
  -v $(pwd):/workspace/Open-Sora \
  -p 7860:7860 \
  --name opensora \
  opensora:latest

# 容器内下载模型权重
pip install "huggingface_hub[cli]"
huggingface-cli download hpcai-tech/Open-Sora-v2 --local-dir ./ckpts

FROM nvidia/cuda:12.1.0-cudnn8-devel-ubuntu22.04

WORKDIR /workspace/Open-Sora

# 系统依赖
RUN apt-get update && apt-get install -y \
    git wget curl build-essential \
    libgl1 libglib2.0-0 \
    && rm -rf /var/lib/apt/lists/*

# Python 依赖
COPY requirements.txt .
RUN pip install -r requirements.txt

# 安装 Open-Sora
COPY . .
RUN pip install -v .

# 安装性能优化器
RUN pip install xformers==0.0.27.post2 --index-url https://download.pytorch.org/whl/cu121
RUN pip install flash-attn --no-build-isolation

EXPOSE 7860
CMD ["/bin/bash"]

# 选项 1：HuggingFace
pip install "huggingface_hub[cli]"
huggingface-cli download hpcai-tech/Open-Sora-v2 --local-dir ./ckpts

# 选项 2：ModelScope（中国用户）
pip install modelscope
modelscope download hpcai-tech/Open-Sora-v2 --local_dir ./ckpts

# 验证下载
ls -la ./ckpts/
# 期望: model.safetensors, config.json, vae/, text_encoder/

# 分离环境安装 ComfyUI
git clone https://github.com/comfyanonymous/ComfyUI.git
cd ComfyUI
pip install -r requirements.txt

# 创建 Open-Sora 自定义节点
mkdir -p custom_nodes/opensora-bridge
cd custom_nodes/opensora-bridge

# custom_nodes/opensora-bridge/opensora_node.py
import subprocess
import torch
import os

class OpenSoraTextToVideo:
    """ComfyUI Open-Sora 文本到视频节点"""
    
    @classmethod
    def INPUT_TYPES(cls):
        return {
            "required": {
                "prompt": ("STRING", {"multiline": True}),
                "resolution": (["256px", "768px"], {"default": "768px"}),
                "num_frames": ("INT", {"default": 65, "min": 17, "max": 129}),
                "steps": ("INT", {"default": 50, "min": 20, "max": 100}),
            }
        }
    
    RETURN_TYPES = ("VIDEO",)
    FUNCTION = "generate_video"
    CATEGORY = "video_generation"
    
    def generate_video(self, prompt, resolution, num_frames, steps):
        # 写提示到 CSV 批量处理
        with open("/tmp/opensora_input.csv", "w") as f:
            f.write(f"id,text\n0,\"{prompt}\"\n")
        
        # 启动推理
        cmd = [
            "torchrun", "--nproc_per_node", "1", "--standalone",
            "scripts/diffusion/inference.py",
            f"configs/diffusion/inference/{resolution}.py",
            "--save-dir", "/tmp/opensora_output",
            "--dataset.data-path", "/tmp/opensora_input.csv",
            "--num_frames", str(num_frames),
        ]
        subprocess.run(cmd, cwd="/workspace/Open-Sora")
        
        video_path = "/tmp/opensora_output/video_0.mp4"
        return (video_path,)

NODE_CLASS_MAPPINGS = {
    "OpenSoraTextToVideo": OpenSoraTextToVideo,
}

NODE_DISPLAY_NAME_MAPPINGS = {
    "OpenSoraTextToVideo": "Open-Sora 文本到视频",
}

# configs/diffusion/inference/t2i2v_768px.py
# 文本到图像到视频配置
model = dict(
    type="STDiT3",
    from_pretrained="./ckpts/model.safetensors",
    enable_flash_attn=True,
    enable_layernorm_kernel=True,
)

vae = dict(
    type="DC-AE",
    from_pretrained="./ckpts/vae",
)

text_encoder = dict(
    type="t5",
    from_pretrained="./ckpts/text_encoder",
)

# FLUX T2I 模型配置
t2i_model = dict(
    type="flux-schnell",
    from_pretrained="black-forest-labs/FLUX.1-schnell",
)

# 采样配置
num_sampling_steps = 50
cfg_scale = 7.5
cfg_channel = 3  # 图像条件尺度

# 安装 Gradio 依赖
pip install gradio spaces

# 启动 Web UI
python gradio/app.py --model-type v2 --checkpoint ./ckpts

# 安装 ColossalAI
pip install colossalai

# 多 GPU 训练启动（8x A100）
torchrun --nproc_per_node 8 --standalone \
    scripts/train.py \
    configs/diffusion/train/stage1.py \
    --data-path /path/to/video/dataset \
    --batch-size 2 \
    --mixed-precision bf16

# 长视频（>65 帧）序列并行
torchrun --nproc_per_node 8 --standalone \
    scripts/train.py \
    configs/diffusion/train/stage2_sp.py \
    --data-path /path/to/video/dataset \
    --sequence-parallel-size 4

# 1. CPU 卸载（省 ~40% VRAM，25% 慢）
torchrun --nproc_per_node 1 --standalone \
    scripts/diffusion/inference.py \
    configs/diffusion/inference/t2i2v_256px.py \
    --prompt "raining, sea" \
    --offload True

# 2. Flash Attention 3（15-20% 加速，无质量损失）
# 先安装：
git clone https://github.com/Dao-AILab/flash-attention
cd flash-attention/hopper
python setup.py install

# 3. 量化推理（INT8，省 50% VRAM）
torchrun --nproc_per_node 1 --standalone \
    scripts/diffusion/inference.py \
    configs/diffusion/inference/t2i2v_256px.py \
    --prompt "raining, sea" \
    --quantization int8

# 4. 混合精度（BF16）
torchrun --nproc_per_node 1 --standalone \
    scripts/diffusion/inference.py \
    configs/diffusion/inference/t2i2v_256px.py \
    --prompt "raining, sea" \
    --mixed-precision bf16

# 高分辨率生成张量并行
torchrun --nproc_per_node 8 --standalone \
    scripts/diffusion/inference.py \
    configs/diffusion/inference/t2i2v_768px.py \
    --save-dir samples \
    --prompt " soaring drone footage captures coastal cliffs" \
    --tp-size 4

# 有效提示结构
prompt = """电影广角镜头金色 retriever 日落沙滩奔跑。
背景海浪破温暖金色小时光照。
相机慢动作跟随狗，捕捉飞毛细节。
高制作价值，变形宽银幕镜头，浅景深。"""

# 避免：模糊或抽象提示
# 坏: "a dog video"
# 好：详细主体 + 动作 + 环境 + 光照 + 相机运动

# docker-compose.prod.yml
version: '3.8'

services:
  opensora:
    build: .
    runtime: nvidia
    environment:
      - NVIDIA_VISIBLE_DEVICES=all
      - CUDA_VISIBLE_DEVICES=0,1,2,3
      - HF_HOME=/workspace/cache
    volumes:
      - ./ckpts:/workspace/Open-Sora/ckpts:ro
      - ./samples:/workspace/Open-Sora/samples
      - huggingface_cache:/workspace/cache