DVC: 面向数据的 Git — ML 流水线数据版本控制与可复现实验 2026 完整指南

# 本地缓存结构
.dvc/cache/
  files/
    md5/
      a1/
        b2c3d4e5f6...  # 实际文件内容

# dvc.yaml — 流水线定义
stages:
  prepare:
    cmd: python src/preprocess.py --input data/raw.csv --output data/processed.csv
    deps:
      - src/preprocess.py
      - data/raw.csv
    outs:
      - data/processed.csv

  train:
    cmd: python src/train.py --data data/processed.csv --model models/model.pkl
    deps:
      - src/train.py
      - data/processed.csv
    outs:
      - models/model.pkl
    params:
      - train.epochs
      - train.lr

# 核心 DVC (最小安装)
pip install dvc

# 带云存储支持
pip install "dvc[s3]"      # AWS S3
pip install "dvc[gs]"      # Google Cloud Storage
pip install "dvc[azure]"   # Azure Blob Storage
pip install "dvc[ssh]"     # SSH/SFTP
pip install "dvc[all]"     # 所有远程后端

dvc --version
# dvc version 3.67.1

cd my-ml-project
git init          # 如果还不是 Git 仓库
dvc init          # 创建 .dvc/ 目录和 .dvcignore
git add .dvc
git commit -m "Initialize DVC"

# 添加单个文件
dvc add data/training_data.csv

# 添加整个目录
dvc add data/images/

# DVC 创建指针文件 (.dvc 文件)
ls data/
# training_data.csv
# training_data.csv.dvc   <- 这个提交到 Git
# .gitignore               <- DVC 将数据添加到 gitignore

git add data/training_data.csv.dvc data/.gitignore
git commit -m "Track training dataset with DVC"

# 从远程拉取数据 (配置远程存储后)
dvc pull

# 或检出特定版本
git checkout v1.0
dvc checkout   # 恢复与 .dvc 指针对应的数据文件

# 添加 S3 作为默认远程
dvc remote add -d myremote s3://my-bucket/dvc-storage

# 使用特定 AWS 配置文件
dvc remote add -d myremote s3://my-bucket/dvc-storage --profile production

# 设置区域
dvc remote modify myremote region us-east-1

# 添加 GCS 远程
dvc remote add -d myremote gs://my-bucket/dvc-storage

# 使用服务账号
dvc remote modify myremote credentialpath /path/to/service-account.json

# 添加 Azure 远程
dvc remote add -d myremote azure://my-container/dvc-storage

# 设置账户名和密钥
dvc remote modify myremote account_name 'myaccount'
dvc remote modify myremote account_key 'mykey'

# 将所有追踪的数据推送到远程
dvc push

# 从远程拉取数据 (团队成员使用)
dvc pull

# 获取特定目标的数据
dvc pull data/training_data.csv

# dvc.yaml
stages:
  prepare:
    cmd: python src/prepare.py --config params.yaml
    deps:
      - src/prepare.py
      - data/raw.csv
    outs:
      - data/prepared/

  featurize:
    cmd: python src/featurize.py --config params.yaml
    deps:
      - src/featurize.py
      - data/prepared/
    outs:
      - data/features/

  train:
    cmd: python src/train.py --config params.yaml
    deps:
      - src/train.py
      - data/features/
    outs:
      - models/model.pkl
    params:
      - train.lr
      - train.epochs
      - train.batch_size
    metrics:
      - metrics.json:
          cache: false

  evaluate:
    cmd: python src/evaluate.py --config params.yaml
    deps:
      - src/evaluate.py
      - models/model.pkl
      - data/features/
    metrics:
      - metrics.json:
          cache: false
    plots:
      - plots/roc_curve.csv

# 运行所有阶段 (仅重新运行变化的阶段)
dvc repro

# 运行特定阶段
dvc repro train

# 可视化流水线
dvc dag

# params.yaml
prepare:
  split: 0.2
  seed: 42

train:
  lr: 0.001
  epochs: 50
  batch_size: 32
  model_type: resnet50

# 运行修改参数的实验
dvc exp run --set-param train.lr=0.01

# 网格搜索运行多个实验
dvc exp run --set-param train.lr=0.1,0.01,0.001

# 列出所有实验
dvc exp show

# 显示带指标的实验表
dvc exp show --no-timestamp --precision 4

# 将成功实验应用到工作区
dvc exp apply exp-abc123

# 推送实验到远程
dvc exp push origin exp-abc123

# dvc.yaml (图表部分)
plots:
  - plots/loss.csv:
      x: step
      y: loss
      title: Training Loss
  - plots/accuracy.csv:
      x: step
      y: accuracy
      title: Validation Accuracy

# 生成并查看图表
dvc plots show

# .github/workflows/ml-pipeline.yml
name: ML Pipeline
on: [push]
jobs:
  train:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install dependencies
        run: |
          pip install dvc[s3]
          pip install -r requirements.txt

      - name: Configure DVC remote
        env:
          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
        run: |
          dvc remote add -d myremote s3://my-bucket/dvc-storage

      - name: Pull data
        run: dvc pull

      - name: Run pipeline
        run: dvc repro

      - name: Upload metrics
        uses: actions/upload-artifact@v4
        with:
          name: metrics
          path: metrics.json

# .gitlab-ci.yml
stages:
  - data
  - train
  - evaluate

variables:
  AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID
  AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY

pull_data:
  stage: data
  image: python:3.11
  script:
    - pip install dvc[s3]
    - dvc remote add -d myremote s3://my-bucket/dvc-storage
    - dvc pull
  artifacts:
    paths:
      - .dvc/
      - data/

train_model:
  stage: train
  image: python:3.11
  dependencies:
    - pull_data
  script:
    - pip install -r requirements.txt
    - dvc repro train
  artifacts:
    paths:
      - models/
      - metrics.json

evaluate_model:
  stage: evaluate
  image: python:3.11
  dependencies:
    - train_model
  script:
    - dvc repro evaluate
    - cat metrics.json

# 仅保留当前 Git 工作区引用的文件
dvc gc --workspace

# 保留所有 Git 分支和标签引用的文件
dvc gc --all-branches --all-tags

# 预览将被删除的内容 (模拟运行)
dvc gc --workspace --dry

# 生产远程 (大多数用户只读)
dvc remote add production s3://prod-bucket/dvc-storage

# 开发远程
dvc remote add -d dev s3://dev-bucket/dvc-storage

# 推送到特定远程
dvc push --remote production

# 导入数据而不复制 (追踪外部 URL)
dvc import-url s3://external-bucket/dataset.csv data/dataset.csv

# 带版本导入 (追踪特定版本)
dvc import-url --rev v1.0 https://github.com/user/repo/data.csv

# 更新导入的数据
dvc update data/dataset.csv

# 使用 reflinks (写时复制) — 最快，不重复占用空间
dvc config cache.type reflink,hardlink,copy

# 验证缓存完整性
dvc cache dir --show

# 检查缓存健康状态
dvc fsck

# 使用 .dvcignore 排除敏感文件
echo "secrets/" >> .dvcignore
echo "*.key" >> .dvcignore

# 远程存储静态加密 (S3 SSE)
dvc remote modify myremote sse AES256

import dvc.api

with dvc.api.open('data/dataset.csv', remote='myremote') as f:
    df = pd.read_csv(f)

dvc remote add -d myremote /mnt/shared-nas/dvc-storage

pip install dvc
cd your-ml-project
dvc init
dvc add your-dataset.csv