Hugging Face 教程¶

from transformers import pipeline

# 文本分类
classifier = pipeline("sentiment-analysis")
result = classifier("I love this product!")
print(result)  # [{'label': 'POSITIVE', 'score': 0.9998}]

# 文本生成
generator = pipeline("text-generation", model="gpt2")
result = generator("The future of AI is", max_length=50)
print(result[0]['generated_text'])

# 问答
qa = pipeline("question-answering")
result = qa(question="What is the capital of France?", 
            context="France is a country in Europe. Its capital is Paris.")
print(result['answer'])  # Paris

# 翻译
translator = pipeline("translation_en_to_fr", model="t5-base")
result = translator("Hello, how are you?")
print(result[0]['translation_text'])

# 图像分类
image_classifier = pipeline("image-classification")
result = image_classifier("cat.jpg")
print(result)

from transformers import AutoTokenizer

# 加载分词器
tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")

# 分词
text = "我喜欢人工智能"
tokens = tokenizer(text)
print(tokens)

# 编码
encoded = tokenizer(text, padding=True, truncation=True, return_tensors="pt")
print(encoded)

# 解码
decoded = tokenizer.decode(encoded['input_ids'][0])
print(decoded)

from transformers import AutoModelForSequenceClassification
import torch

# 加载模型
model = AutoModelForSequenceClassification.from_pretrained(
    "bert-base-chinese", 
    num_labels=2
)

# 推理
inputs = tokenizer("测试文本", return_tensors="pt")
outputs = model(**inputs)
predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)
print(predictions)

from huggingface_hub import list_models

# 搜索模型
models = list_models(search="sentiment-analysis", limit=5)
for model in models:
    print(f"{model.id} - {model.tags}")

from transformers import AutoModel

# 下载模型
model = AutoModel.from_pretrained("bert-base-chinese")

# 下载到指定目录
model = AutoModel.from_pretrained(
    "bert-base-chinese",
    cache_dir="./my_models"
)

from huggingface_hub import HfApi

api = HfApi()

# 上传模型
api.upload_folder(
    folder_path="./my_model",
    repo_id="username/my-model",
    repo_type="model"
)

from datasets import load_dataset

# 加载内置数据集
dataset = load_dataset("imdb")
print(dataset)

# 加载自定义数据集
dataset = load_dataset("csv", data_files="my_data.csv")

# 加载json数据
dataset = load_dataset("json", data_files="my_data.json")

# 查看数据
print(dataset['train'][0])

# 数据集分割
train_dataset = dataset['train']
test_dataset = dataset['test']

# 数据处理
def preprocess(examples):
    return tokenizer(examples['text'], padding=True, truncation=True)

tokenized_dataset = train_dataset.map(preprocess, batched=True)

# 保存数据集
dataset.save_to_disk("./my_dataset")

from transformers import (
    AutoModelForSequenceClassification,
    AutoTokenizer,
    TrainingArguments,
    Trainer
)
from datasets import load_dataset

# 加载数据和模型
dataset = load_dataset("imdb")
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased")

# 数据预处理
def tokenize_function(examples):
    return tokenizer(examples["text"], padding="max_length", truncation=True)

tokenized_datasets = dataset.map(tokenize_function, batched=True)

# 训练参数
training_args = TrainingArguments(
    output_dir="./results",
    num_train_epochs=3,
    per_device_train_batch_size=16,
    per_device_eval_batch_size=64,
    warmup_steps=500,
    weight_decay=0.01,
    logging_dir="./logs",
    logging_steps=10,
    evaluation_strategy="epoch"
)

# 创建Trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=tokenized_datasets["train"],
    eval_dataset=tokenized_datasets["test"]
)

# 训练
trainer.train()

# 保存模型
trainer.save_model("./my_model")

import requests

API_URL = "https://api-inference.huggingface.co/models/bert-base-chinese"
headers = {"Authorization": "Bearer YOUR_TOKEN"}

def query(payload):
    response = requests.post(API_URL, headers=headers, json=payload)
    return response.json()

output = query({
    "inputs": "这是一个测试句子"
})
print(output)

from transformers import pipeline

# 本地推理
classifier = pipeline("sentiment-analysis", model="./my_model")
result = classifier("测试文本")
print(result)

from transformers import pipeline
import pandas as pd

# 创建情感分析器
analyzer = pipeline("sentiment-analysis", 
                   model="nlptown/bert-base-multilingual-uncased-sentiment")

# 分析多条评论
reviews = [
    "这个产品非常好用，推荐购买！",
    "质量太差了，不值这个价格",
    "一般般，没什么特别的"
]

results = []
for review in reviews:
    result = analyzer(review)[0]
    results.append({
        "review": review,
        "sentiment": result['label'],
        "score": result['score']
    })

df = pd.DataFrame(results)
print(df)

from transformers import pipeline

summarizer = pipeline("summarization", model="facebook/bart-large-cnn")

text = """
Natural language processing (NLP) is a subfield of linguistics, computer 
science, and artificial intelligence concerned with the interactions between 
computers and human language. The goal is to enable computers to understand, 
interpret, and generate human language in a valuable way.
"""

summary = summarizer(text, max_length=50, min_length=10)
print(summary[0]['summary_text'])