Hugging Face Transformers：完整开发者指南（2025 版）

pip install transformers torch

pip install transformers datasets accelerate peft trl

pip install torch --index-url https://download.pytorch.org/whl/cu121

import torch
print(torch.cuda.is_available())  # True

from transformers import pipeline
classifier = pipeline("sentiment-analysis")
result = classifier("这部电影太棒了！")
# [{'label': 'POSITIVE', 'score': 0.9998}]

ner = pipeline("ner", aggregation_strategy="simple")
result = ner("苹果公司由史蒂夫·乔布斯在加州创立。")
# [{'entity_group': 'ORG', 'word': '苹果公司'}, ...]

qa = pipeline("question-answering")
result = qa(
    question="法国的首都是哪里？",
    context="巴黎是法国的首都。"
)
# {'answer': '巴黎', 'score': 0.99}

generator = pipeline("text-generation", model="gpt2")
result = generator("人工智能的未来是", max_length=30)

translator = pipeline("translation_en_to_de", model="t5-base")
result = translator("你好，最近怎么样？")

from transformers import AutoModel, AutoTokenizer

model_name = "bert-base-uncased"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModel.from_pretrained(model_name)

from datasets import load_dataset

dataset = load_dataset("imdb")
def tokenize_function(examples):
    return tokenizer(examples["text"], padding="max_length", truncation=True)

tokenized = dataset.map(tokenize_function, batched=True)

from peft import LoraConfig, get_peft_model

lora_config = LoraConfig(
    r=16,
    lora_alpha=32,
    target_modules=["q_proj", "v_proj"],
    task_type="CAUSAL_LM"
)
model = get_peft_model(model, lora_config)

from transformers import BitsAndBytesConfig
bnb_config = BitsAndBytesConfig(load_in_4bit=True)
model = AutoModelForCausalLM.from_pretrained(
    "meta-llama/Llama-2-7b",
    quantization_config=bnb_config
)

torch.onnx.export(
    model,
    (torch.zeros(1, 128, dtype=torch.long),),
    "model.onnx",
    input_names=["input_ids"],
    output_names=["logits"]
)