自然语言处理（NLP）¶

import re
import nltk
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords

# 下载资源
nltk.download('punkt')
nltk.download('stopwords')

# 文本清洗
def clean_text(text):
    # 转小写
    text = text.lower()
    # 移除标点
    text = re.sub(r'[^\w\s]', '', text)
    # 移除数字
    text = re.sub(r'\d+', '', text)
    return text

# 分词
text = "Natural Language Processing is amazing!"
tokens = word_tokenize(text)
print(tokens)

# 移除停用词
stop_words = set(stopwords.words('english'))
filtered = [w for w in tokens if w.lower() not in stop_words]
print(filtered)

import numpy as np

vocab = {'apple': 0, 'banana': 1, 'orange': 2}
one_hot = np.zeros((len(vocab), len(vocab)))
for word, idx in vocab.items():
    one_hot[idx, idx] = 1

print(one_hot)

from sklearn.feature_extraction.text import TfidfVectorizer

documents = [
    "Machine learning is great",
    "Natural language processing is useful",
    "Machine learning and NLP are related"
]

vectorizer = TfidfVectorizer()
tfidf_matrix = vectorizer.fit_transform(documents)

print(vectorizer.get_feature_names_out())
print(tfidf_matrix.toarray())

from gensim.models import Word2Vec

sentences = [
    ['machine', 'learning', 'is', 'great'],
    ['natural', 'language', 'processing', 'is', 'useful'],
    ['machine', 'learning', 'and', 'nlp', 'are', 'related']
]

model = Word2Vec(sentences, vector_size=100, window=5, min_count=1)

# 获取词向量
vector = model.wv['machine']
print(f"Word vector shape: {vector.shape}")

from sklearn.naive_bayes import MultinomialNB
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.pipeline import Pipeline

# 训练情感分类器
train_texts = ["Great product!", "Terrible experience", "Love it", "Hate it"]
train_labels = [1, 0, 1, 0]  # 1=正面, 0=负面

# 构建Pipeline
sentiment_clf = Pipeline([
    ('tfidf', TfidfVectorizer(max_features=1000)),
    ('clf', MultinomialNB())
])

sentiment_clf.fit(train_texts, train_labels)

# 预测
test_texts = ["This is amazing!", "Worst purchase ever"]
predictions = sentiment_clf.predict(test_texts)
print(predictions)

from sklearn.datasets import fetch_20newsgroups
from sklearn.svm import LinearSVC
from sklearn.metrics import classification_report

# 加载数据
categories = ['sci.space', 'rec.sport.baseball', 'comp.graphics']
train = fetch_20newsgroups(subset='train', categories=categories)
test = fetch_20newsgroups(subset='test', categories=categories)

# Pipeline
clf = Pipeline([
    ('tfidf', TfidfVectorizer(stop_words='english')),
    ('clf', LinearSVC())
])

clf.fit(train.data, train.target)

# 评估
predictions = clf.predict(test.data)
print(classification_report(test.target, predictions, 
                          target_names=categories))

import spacy

# 加载模型
nlp = spacy.load("en_core_web_sm")

# NER
text = "Apple Inc. was founded by Steve Jobs in California in 1976."
doc = nlp(text)

for ent in doc.ents:
    print(f"{ent.text}: {ent.label_}")

from transformers import pipeline

# 预训练情感分析模型
sentiment_analyzer = pipeline("sentiment-analysis")

# 分析文本
results = sentiment_analyzer([
    "I love this product!",
    "This is terrible.",
    "It's okay, nothing special."
])

for text, result in zip(texts, results):
    print(f"{text}: {result['label']} ({result['score']:.2f})")

from transformers import GPT2LMHeadModel, GPT2Tokenizer

# 加载模型
tokenizer = GPT2Tokenizer.from_pretrained('gpt2')
model = GPT2LMHeadModel.from_pretrained('gpt2')

# 生成文本
input_text = "Machine learning is"
input_ids = tokenizer.encode(input_text, return_tensors='pt')

output = model.generate(
    input_ids,
    max_length=100,
    num_return_sequences=1,
    temperature=0.7
)

generated_text = tokenizer.decode(output[0], skip_special_tokens=True)
print(generated_text)

from transformers import MarianMTModel, MarianTokenizer

# 英译中
model_name = 'Helsinki-NLP/opus-mt-en-zh'
tokenizer = MarianTokenizer.from_pretrained(model_name)
model = MarianMTModel.from_pretrained(model_name)

text = "Hello, how are you?"
inputs = tokenizer(text, return_tensors="pt", padding=True)
translated = model.generate(**inputs)
result = tokenizer.decode(translated[0], skip_special_tokens=True)
print(result)

from transformers import pipeline

qa_pipeline = pipeline("question-answering")

context = """
Python is a high-level programming language created by Guido van Rossum.
It was first released in 1991. Python emphasizes code readability and
allows programmers to express concepts in fewer lines of code.
"""

question = "Who created Python?"
result = qa_pipeline(question=question, context=context)
print(f"Answer: {result['answer']}")
print(f"Score: {result['score']:.2f}")

from summa import summarizer

text = """
Natural language processing (NLP) is a subfield of linguistics, computer 
science, and artificial intelligence concerned with the interactions between 
computers and human language. The goal is to enable computers to understand, 
interpret, and generate human language in a valuable way.
"""

# 生成摘要
summary = summarizer.summarize(text, ratio=0.5)
print(summary)