Hugging Face Transformers库使用指南2025：开发者完整教程

# 最小安装（仅PyTorch支持）
pip install transformers torch

# 完整安装（含所有框架和依赖）
pip install transformers[torch] datasets accelerate

# 如果需要使用SentencePiece模型（如XLM-RoBERTa）
pip install sentencepiece

# NVIDIA GPU - 安装CUDA版本的PyTorch
pip install torch --index-url https://download.pytorch.org/whl/cu121

# 验证GPU可用
python -c "import torch; print(torch.cuda.is_available())"

from transformers import pipeline

classifier = pipeline("sentiment-analysis", 
                      model="distilbert-base-uncased-finetuned-sst-2-english")
result = classifier("I love using the Transformers library!")
# [{'label': 'POSITIVE', 'score': 0.9998}]

ner = pipeline("ner", model="dslim/bert-base-NER", aggregation_strategy="simple")
result = ner("Apple Inc. is planning to open a new office in Paris.")
# 识别出组织（Apple Inc.）和地点（Paris）

qa = pipeline("question-answering", model="deepset/roberta-base-squad2")
context = "Hugging Face was founded in 2016 by Clement Delangue and Julien Chaumond."
result = qa(question="Who founded Hugging Face?", context=context)
# {'answer': 'Clement Delangue and Julien Chaumond', 'score': 0.98}

generator = pipeline("text-generation", model="gpt2")
result = generator("Once upon a time", max_length=50, num_return_sequences=1)

summarizer = pipeline("summarization", model="facebook/bart-large-cnn")
text = """你的长文本..."""
result = summarizer(text, max_length=100, min_length=30)

translator = pipeline("translation_en_to_de", model="Helsinki-NLP/opus-mt-en-de")
result = translator("Hello world!")

from transformers import AutoTokenizer, AutoModelForSequenceClassification

tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")
model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-chinese", 
    num_labels=2
)

# 使用
text = "这是一个测试句子"
inputs = tokenizer(text, return_tensors="pt", padding=True, truncation=True)
outputs = model(**inputs)
predictions = outputs.logits.argmax(dim=-1)

from transformers import AutoConfig

# 查看模型配置
config = AutoConfig.from_pretrained("bert-base-chinese")
print(config.hidden_size)      # 768
print(config.num_hidden_layers) # 12

# 保存模型到本地
model.save_pretrained("./my_model")
tokenizer.save_pretrained("./my_model")

# 从本地加载
model = AutoModelForSequenceClassification.from_pretrained("./my_model")

from transformers import AutoTokenizer

tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")

# 查看词汇表大小
print(tokenizer.vocab_size)  # 21128

# 编码文本
text = "Hugging Face很棒"
encoded = tokenizer(text, padding=True, truncation=True, max_length=512)
# input_ids, attention_mask, token_type_ids

# 解码回文本
decoded = tokenizer.decode(encoded["input_ids"])

# 批量编码
texts = ["第一条文本", "第二条文本"]
batch = tokenizer(texts, padding=True, truncation=True, return_tensors="pt")

from transformers import (
    AutoTokenizer, AutoModelForSequenceClassification,
    TrainingArguments, Trainer
)
from datasets import load_dataset

# 加载数据和模型
dataset = load_dataset("imdb")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-uncased", num_labels=2
)

# 数据预处理
def tokenize(batch):
    return tokenizer(batch["text"], padding=True, truncation=True)

dataset = dataset.map(tokenize, batched=True)

# 训练配置
training_args = TrainingArguments(
    output_dir="./results",
    evaluation_strategy="epoch",
    learning_rate=2e-5,
    per_device_train_batch_size=16,
    num_train_epochs=3,
    weight_decay=0.01,
)

# 训练
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=dataset["train"].shuffle(seed=42).select(range(2000)),
    eval_dataset=dataset["test"].shuffle(seed=42).select(range(500)),
)
trainer.train()

from peft import LoraConfig, get_peft_model

lora_config = LoraConfig(
    r=16,                    # 低秩维度
    lora_alpha=32,           # 缩放参数
    target_modules=["q_proj", "v_proj"],  # 应用LoRA的层
    lora_dropout=0.05,
    bias="none",
    task_type="CAUSAL_LM"
)

model = get_peft_model(model, lora_config)
model.print_trainable_parameters()
# 可训练参数通常只有全量微调的0.1%-1%

from transformers import BitsAndBytesConfig
import torch

bnb_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.float16
)

model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-2-7b-hf",
    quantization_config=bnb_config,
    device_map="auto"
)

from transformers import AutoModelForSequenceClassification, AutoTokenizer
import torch

model = AutoModelForSequenceClassification.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")
tokenizer = AutoTokenizer.from_pretrained("distilbert-base-uncased-finetuned-sst-2-english")

# 导出为ONNX
torch.onnx.export(
    model,
    tokenizer("Hello", return_tensors="pt")["input_ids"],
    "model.onnx",
    input_names=["input_ids"],
    output_names=["output"],
    dynamic_axes={"input_ids": {0: "batch", 1: "sequence"}}
)

import requests

API_URL = "https://api-inference.huggingface.co/models/bert-base-chinese"
headers = {"Authorization": f"Bearer {YOUR_TOKEN}"}

def query(payload):
    response = requests.post(API_URL, headers=headers, json=payload)
    return response.json()

result = query({"inputs": "这是一个测试"})