经典机器学习算法详解¶
概述¶
本教程深入讲解经典的机器学习算法，包括原理、数学推导和Python实现，帮助你真正理解算法本质。
监督学习算法¶
1. 线性回归（Linear Regression）¶
原理¶
损失函数¶
# 均方误差（MSE）
def mse_loss(y_true, y_pred):
return np.mean((y_true - y_pred) ** 2)
# 梯度下降更新
def gradient_descent(X, y, lr=0.01, epochs=1000):
n_samples, n_features = X.shape
w = np.zeros(n_features)
b = 0
for _ in range(epochs):
y_pred = X @ w + b
# 计算梯度
dw = (1/n_samples) * (X.T @ (y_pred - y))
db = (1/n_samples) * np.sum(y_pred - y)
# 更新参数
w -= lr * dw
b -= lr * db
return w, b
正则化¶
# L2正则化（Ridge）
def ridge_loss(y_true, y_pred, w, lambda_reg=0.1):
mse = np.mean((y_true - y_pred) ** 2)
l2_penalty = lambda_reg * np.sum(w ** 2)
return mse + l2_penalty
# L1正则化（Lasso）- 产生稀疏解
def lasso_loss(y_true, y_pred, w, lambda_reg=0.1):
mse = np.mean((y_true - y_pred) ** 2)
l1_penalty = lambda_reg * np.sum(np.abs(w))
return mse + l1_penalty
2. 逻辑回归（Logistic Regression）¶
原理¶
# 核心思想
虽然名字叫"回归"，实际用于二分类问题。使用Sigmoid函数将线性输出映射到概率。
# Sigmoid函数
σ(z) = 1 / (1 + e⁻ᶻ)
输出范围：(0, 1)，可解释为概率
数学推导¶
import numpy as np
def sigmoid(z):
return 1 / (1 + np.exp(-z))
# 对数似然损失（交叉熵）
def logistic_loss(y_true, y_pred):
epsilon = 1e-15 # 防止log(0)
y_pred = np.clip(y_pred, epsilon, 1 - epsilon)
return -np.mean(y_true * np.log(y_pred) + (1 - y_true) * np.log(1 - y_pred))
多分类扩展¶
# Softmax回归（多分类逻辑回归）
def softmax(z):
exp_z = np.exp(z - np.max(z, axis=1, keepdims=True))
return exp_z / np.sum(exp_z, axis=1, keepdims=True)
# 交叉熵损失
def cross_entropy_loss(y_true, y_pred):
return -np.mean(np.sum(y_true * np.log(y_pred + 1e-15), axis=1))
3. 支持向量机（SVM）¶
原理¶
# 核心思想
找到一个超平面，使得两类样本之间的间隔（margin）最大化。
# 数学表达
最大化间隔：2 / ||w||
等价于最小化：(1/2)||w||²
约束条件：yᵢ(wᵀxᵢ + b) ≥ 1
软间隔SVM¶
# 带松弛变量的SVM
# 最小化：(1/2)||w||² + CΣξᵢ
# 约束：yᵢ(wᵀxᵢ + b) ≥ 1 - ξᵢ, ξᵢ ≥ 0
def svm_loss(X, y, w, b, C=1.0):
n_samples = X.shape[0]
distances = y * (X @ w + b)
hinge_loss = np.maximum(0, 1 - distances)
return 0.5 * np.dot(w, w) + C * np.mean(hinge_loss)
核技巧¶
# 常用核函数
# 线性核
def linear_kernel(x1, x2):
return np.dot(x1, x2)
# 多项式核
def polynomial_kernel(x1, x2, degree=3):
return (np.dot(x1, x2) + 1) ** degree
# RBF核（高斯核）
def rbf_kernel(x1, x2, gamma=0.1):
return np.exp(-gamma * np.linalg.norm(x1 - x2) ** 2)
4. 决策树（Decision Tree）¶
原理¶
基尼系数¶
def gini_impurity(y):
classes, counts = np.unique(y, return_counts=True)
probabilities = counts / len(y)
return 1 - np.sum(probabilities ** 2)
# 信息增益
def entropy(y):
classes, counts = np.unique(y, return_counts=True)
probabilities = counts / len(y)
return -np.sum(probabilities * np.log2(probabilities + 1e-10))
def information_gain(y_parent, y_left, y_right):
n_total = len(y_parent)
n_left, n_right = len(y_left), len(y_right)
entropy_parent = entropy(y_parent)
entropy_children = (n_left/n_total * entropy(y_left) +
n_right/n_total * entropy(y_right))
return entropy_parent - entropy_children
5. 随机森林（Random Forest）¶
原理¶
实现¶
from sklearn.ensemble import RandomForestClassifier
from sklearn.tree import DecisionTreeClassifier
class SimpleRandomForest:
def __init__(self, n_trees=100, max_features='sqrt'):
self.n_trees = n_trees
self.max_features = max_features
self.trees = []
def fit(self, X, y):
for _ in range(self.n_trees):
# Bootstrap采样
indices = np.random.choice(len(X), len(X), replace=True)
X_bootstrap, y_bootstrap = X[indices], y[indices]
# 训练单棵树
tree = DecisionTreeClassifier(max_features=self.max_features)
tree.fit(X_bootstrap, y_bootstrap)
self.trees.append(tree)
def predict(self, X):
# 多数投票
predictions = np.array([tree.predict(X) for tree in self.trees])
return np.apply_along_axis(
lambda x: np.bincount(x.astype(int)).argmax(),
0, predictions
)
6. XGBoost¶
原理¶
简化实现¶
def xgboost_objective(y_true, y_pred, gamma=0, lambda_reg=1):
# 目标函数 = 损失函数 + 正则化
n = len(y_true)
# 一阶和二阶导数
gradient = y_pred - y_true # 简化的梯度
hessian = np.ones(n) # 简化的Hessian
# 正则化项
tree_complexity = 0 # 简化
reg = lambda_reg * tree_complexity
return np.sum(gradient ** 2 / (hessian + 1e-10)) + reg
无监督学习算法¶
1. K-Means聚类¶
class KMeans:
def __init__(self, k=3, max_iters=100):
self.k = k
self.max_iters = max_iters
def fit(self, X):
# 随机初始化聚类中心
indices = np.random.choice(len(X), self.k, replace=False)
self.centroids = X[indices]
for _ in range(self.max_iters):
# 分配簇
distances = np.array([np.linalg.norm(X - c, axis=1)
for c in self.centroids])
self.labels = np.argmin(distances, axis=0)
# 更新中心
new_centroids = np.array([X[self.labels == i].mean(axis=0)
for i in range(self.k)])
if np.allclose(self.centroids, new_centroids):
break
self.centroids = new_centroids
def predict(self, X):
distances = np.array([np.linalg.norm(X - c, axis=1)
for c in self.centroids])
return np.argmin(distances, axis=0)
2. DBSCAN¶
def dbscan(X, eps=0.5, min_samples=5):
n_samples = len(X)
labels = -1 * np.ones(n_samples) # -1表示未分类
cluster_id = 0
for i in range(n_samples):
if labels[i] != -1:
continue
# 找邻域内的点
neighbors = [j for j in range(n_samples)
if np.linalg.norm(X[i] - X[j]) < eps]
if len(neighbors) < min_samples:
labels[i] = -2 # 标记为噪声点
continue
# 创建新簇
labels[i] = cluster_id
seed_set = set(neighbors)
while seed_set:
j = seed_set.pop()
if labels[j] == -2:
labels[j] = cluster_id
if labels[j] != -1:
continue
labels[j] = cluster_id
new_neighbors = [k for k in range(n_samples)
if np.linalg.norm(X[j] - X[k]) < eps]
if len(new_neighbors) >= min_samples:
seed_set.update(new_neighbors)
cluster_id += 1
return labels
算法选择指南¶
# 算法选择流程
数据量小（<1K）
→ 简单模型：逻辑回归、SVM
数据量中（1K-100K）
→ 树模型：随机森林、XGBoost
数据量大（>100K）
→ 线性模型、神经网络
特征维度高
→ 线性模型、SVM
需要可解释性
→ 决策树、线性回归
追求最高精度
→ XGBoost、神经网络