# 手撕卷积神经网络+反向传播梯度更新

> 发布于：2025-07-13 | 分类：笔记, 技能分享 | 原文：https://www.imbailey.cn/post/cnn-backpropagation-gradient-update-from-scratch

# 手撕卷积神经网络

一个简单的卷积神经网络，就4层，分别是InputLayer, conv_layer, hidden_layer(全连接层 + ReLU激活), output_layer

```
import numpy as np

class InputLayer:
    """输入层 - 简单的数据传递层"""
    def __init__(self):
        pass
    
    def forward(self, x):
        """前向传播：直接传递输入"""
        self.x = x
        return x
    
    def backward(self, dout):
        """反向传播：直接传递梯度"""
        return dout
    
    def update_params(self):
        """输入层没有参数需要更新"""
        pass


class HiddenLayer:
    """隐藏层 - 全连接层 + ReLU激活"""
    def __init__(self, input_size, output_size, learning_rate=0.001):
        self.learning_rate = learning_rate
        
        # 初始化权重和偏置
        self.weights = np.random.randn(input_size, output_size) * 0.01
        self.bias = np.zeros((1, output_size))
        
        # 梯度缓存
        self.dweights = np.zeros_like(self.weights)
        self.dbias = np.zeros_like(self.bias)
        
        # 前向传播缓存
        self.x = None
        self.z = None  # 激活前的值
        self.a = None  # 激活后的值
        self.original_shape = None  # 保存原始形状以便反向传播时恢复
    
    def forward(self, x):
        """前向传播"""
        # 保存原始形状
        self.original_shape = x.shape
        
        # 如果输入是4D张量(来自卷积层)，需要flatten
        if len(x.shape) == 4:
            batch_size = x.shape[0]
            x = x.reshape(batch_size, -1)
        
        self.x = x
        # 线性变换: z = x * W + b
        self.z = np.dot(x, self.weights) + self.bias
        # ReLU激活: a = max(0, z)
        self.a = np.maximum(0, self.z)
        return self.a
    
    def backward(self, dout):
        """反向传播"""
        batch_size = self.x.shape[0]
        
        # ReLU梯度
        da_dz = (self.z > 0).astype(float)
        dz = dout * da_dz
        
        # 计算梯度
        self.dweights = np.dot(self.x.T, dz)
        self.dbias = np.sum(dz, axis=0, keepdims=True)
        dx = np.dot(dz, self.weights.T)
        
        # 如果原始输入是4D，需要恢复形状
        if len(self.original_shape) == 4:
            dx = dx.reshape(self.original_shape)
        
        return dx
    
    def update_params(self):
        """更新参数"""
        self.weights -= self.learning_rate * self.dweights
        self.bias -= self.learning_rate * self.dbias


class OutputLayer:
    """输出层 - 全连接层 + Softmax激活"""
    def __init__(self, input_size, output_size, learning_rate=0.001):
        self.learning_rate = learning_rate
        
        # 初始化权重和偏置
        self.weights = np.random.randn(input_size, output_size) * 0.01
        self.bias = np.zeros((1, output_size))
        
        # 梯度缓存
        self.dweights = np.zeros_like(self.weights)
        self.dbias = np.zeros_like(self.bias)
        
        # 前向传播缓存
        self.x = None
        self.z = None
        self.a = None
    
    def forward(self, x):
        """前向传播"""
        # 如果输入是4D张量，需要flatten
        if len(x.shape) == 4:
            batch_size = x.shape[0]
            x = x.reshape(batch_size, -1)
        
        self.x = x
        # 线性变换
        self.z = np.dot(x, self.weights) + self.bias
        # Softmax激活
        self.a = self.softmax(self.z)
        return self.a
    
    def softmax(self, z):
        """Softmax激活函数"""
        exp_z = np.exp(z - np.max(z, axis=1, keepdims=True))  # 防止溢出
        return exp_z / np.sum(exp_z, axis=1, keepdims=True)
    
    def backward(self, dout):
        """反向传播"""
        batch_size = self.x.shape[0]
        
        # 对于softmax + 交叉熵，梯度简化为 dz = y_pred - y_true
        dz = dout
        
        # 计算梯度
        self.dweights = np.dot(self.x.T, dz)
        self.dbias = np.sum(dz, axis=0, keepdims=True)
        dx = np.dot(dz, self.weights.T)
        
        return dx
    
    def update_params(self):
        """更新参数"""
        self.weights -= self.learning_rate * self.dweights
        self.bias -= self.learning_rate * self.dbias


class Conv2D:
    def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0, learning_rate=0.001):
        self.in_channels = in_channels 
        self.out_channels = out_channels
        self.kernel_size = kernel_size
        self.stride = stride
        self.padding = padding
        self.learning_rate = learning_rate

        # 初始化权重和偏置
        self.weights = np.random.randn(out_channels, in_channels, kernel_size, kernel_size) * 0.01
        self.bias = np.zeros((out_channels, 1))

        # 梯度缓存
        self.dweights = np.zeros_like(self.weights)
        self.dbias = np.zeros_like(self.bias)
        self.x = None

    def forward(self, x):
        self.x = x
        batch_size, in_channels, in_height, in_width = x.shape

        out_height = (in_height + 2 * self.padding - self.kernel_size) // self.stride + 1
        out_width = (in_width + 2 * self.padding - self.kernel_size) // self.stride + 1

        x_padded = np.pad(x, ((0, 0), (0, 0), (self.padding, self.padding), (self.padding, self.padding)), mode='constant')
        output = np.zeros((batch_size, self.out_channels, out_height, out_width))

        for b in range(batch_size):
            for oc in range(self.out_channels):
                for h in range(out_height):
                    for w in range(out_width):
                        h_start = h * self.stride
                        h_end = h_start + self.kernel_size
                        w_start = w * self.stride
                        w_end = w_start + self.kernel_size

                        window = x_padded[b, :, h_start:h_end, w_start:w_end]
                        output[b, oc, h, w] = np.sum(window * self.weights[oc]) + self.bias[oc]
        return output
    
    def backward(self, dout):
        batch_size, out_channels, out_height, out_width = dout.shape
        _, _, in_height, in_width = self.x.shape

        x_padded = np.pad(self.x, ((0, 0), (0, 0), (self.padding, self.padding), (self.padding, self.padding)), mode='constant')
        dx_padded = np.zeros_like(x_padded)
        self.dweights = np.zeros_like(self.weights)
        self.dbias = np.zeros_like(self.bias)

        for b in range(batch_size):
            for oc in range(out_channels):
                for h in range(out_height):
                    for w in range(out_width):
                        h_start = h * self.stride
                        h_end = h_start + self.kernel_size
                        w_start = w * self.stride
                        w_end = w_start + self.kernel_size

                        grad = dout[b, oc, h, w]
                        self.dweights[oc] += grad * x_padded[b, :, h_start:h_end, w_start:w_end]
                        self.dbias[oc] += grad
                        dx_padded[b, :, h_start:h_end, w_start:w_end] += grad * self.weights[oc]

        # 移除填充
        if self.padding > 0:
            dx = dx_padded[:, :, self.padding:-self.padding, self.padding:-self.padding]
        else:
            dx = dx_padded
        
        return dx
    
    def update_params(self):
        """更新参数"""
        self.weights -= self.learning_rate * self.dweights
        self.bias -= self.learning_rate * self.dbias


def loss_function(y_pred, y_true):
    """交叉熵损失函数"""
    batch_size = y_pred.shape[0]
    # 防止log(0)
    y_pred_clipped = np.clip(y_pred, 1e-15, 1 - 1e-15)
    # 计算交叉熵损失
    loss = -np.sum(y_true * np.log(y_pred_clipped)) / batch_size
    return loss


def loss_gradient(y_pred, y_true):
    """交叉熵损失的梯度"""
    # 对于softmax + 交叉熵，梯度为 y_pred - y_true
    return y_pred - y_true


if __name__ == "__main__":
    # 示例数据
    x0 = np.random.randn(2, 1, 28, 28)  # batch_size=2, channels=1, height=28, width=28
    y = np.array([[1, 0, 0, 0, 0, 0, 0, 0, 0, 0],  # one-hot编码
                  [0, 1, 0, 0, 0, 0, 0, 0, 0, 0]])
    
    # 创建网络层 - 修正网络架构
    input_layer = InputLayer()
    conv_layer = Conv2D(in_channels=1, out_channels=32, kernel_size=3, padding=1)
    hidden_layer = HiddenLayer(input_size=32*28*28, output_size=64)  # 32通道*28*28
    output_layer = OutputLayer(input_size=64, output_size=10)  # 10个类别
    
    print("Network architecture:")
    print("Input -> Conv2D -> HiddenLayer -> OutputLayer")
    print(f"Input shape: {x0.shape}")
    
    # 前向传播
    x1 = input_layer.forward(x0)       # (2, 1, 28, 28)
    print(f"After input_layer: {x1.shape}")
    
    x2 = conv_layer.forward(x1)        # (2, 32, 28, 28)
    print(f"After conv_layer: {x2.shape}")
    
    x3 = hidden_layer.forward(x2)      # (2, 64)
    print(f"After hidden_layer: {x3.shape}")
    
    y_pred = output_layer.forward(x3)  # (2, 10)
    print(f"After output_layer: {y_pred.shape}")

    # 计算损失和梯度
    loss = loss_function(y_pred, y)
    dout = loss_gradient(y_pred, y)

    print(f"\nLoss: {loss:.4f}")
    print(f"Predictions:\n{y_pred}")

    # 反向传播
    dx3 = output_layer.backward(dout)     
    dx2 = hidden_layer.backward(dx3)      
    dx1 = conv_layer.backward(dx2)        
    dx0 = input_layer.backward(dx1)       

    # 更新参数
    output_layer.update_params()
    hidden_layer.update_params()
    conv_layer.update_params()
    input_layer.update_params()

    print("Training step completed!")

```


# 反向传播梯度更新推导


以下是四层网络中每层的反向传播公式（以损失函数 \( L \) 为起点，使用偏导数符号）：


### OutputLayer
**正向传播公式**：  
1. \( Z = XW + b \)  
2. \( A = \text{Softmax}(Z) \)  
3. \( L = \text{CrossEntropy}(A, Y) \)

**反向传播公式**（链式法则展开）：  
1. \( \frac{\partial L}{\partial Z} = \frac{\partial L}{\partial A} \cdot \frac{\partial A}{\partial Z} = A - Y \)（Softmax+交叉熵的梯度简化）  
2. \( \frac{\partial L}{\partial W} = \frac{\partial L}{\partial Z} \cdot \frac{\partial Z}{\partial W} = X^T \cdot \frac{\partial L}{\partial Z} \)  
3. \( \frac{\partial L}{\partial b} = \frac{\partial L}{\partial Z} \cdot \frac{\partial Z}{\partial b} = \sum \frac{\partial L}{\partial Z} \)  
4. \( \frac{\partial L}{\partial X} = \frac{\partial L}{\partial Z} \cdot \frac{\partial Z}{\partial X} = \frac{\partial L}{\partial Z} \cdot W^T \)


### HiddenLayer
**正向传播公式**：  
1. \( Z = XW + b \)  
2. \( A = \text{ReLU}(Z) \)

**反向传播公式**（链式法则展开）：  
1. \( \frac{\partial L}{\partial Z} = \frac{\partial L}{\partial A} \cdot \frac{\partial A}{\partial Z} = \frac{\partial L}{\partial A} \cdot \text{ReLU}'(Z) \)  
2. \( \frac{\partial L}{\partial W} = \frac{\partial L}{\partial Z} \cdot \frac{\partial Z}{\partial W} = X^T \cdot \frac{\partial L}{\partial Z} \)  
3. \( \frac{\partial L}{\partial b} = \frac{\partial L}{\partial Z} \cdot \frac{\partial Z}{\partial b} = \sum \frac{\partial L}{\partial Z} \)  
4. \( \frac{\partial L}{\partial X} = \frac{\partial L}{\partial Z} \cdot \frac{\partial Z}{\partial X} = \frac{\partial L}{\partial Z} \cdot W^T \)


### Conv2D
**正向传播公式**：  
1. \( Z = X \star W + b \)（\( \star \) 表示卷积操作）

**反向传播公式**（链式法则展开）：  
1. \( \frac{\partial L}{\partial W} = \frac{\partial L}{\partial Z} \cdot \frac{\partial Z}{\partial W} = X \star_{\text{rot}} \frac{\partial L}{\partial Z} \)（旋转卷积，等价于 \( X \) 与 \( \frac{\partial L}{\partial Z} \) 的卷积）  
2. \( \frac{\partial L}{\partial b} = \frac{\partial L}{\partial Z} \cdot \frac{\partial Z}{\partial b} = \sum \frac{\partial L}{\partial Z} \)  
3. \( \frac{\partial L}{\partial X} = \frac{\partial L}{\partial Z} \cdot \frac{\partial Z}{\partial X} = \frac{\partial L}{\partial Z} \star W_{\text{rot}} \)（旋转权重卷积，等价于 \( \frac{\partial L}{\partial Z} \) 与旋转后的 \( W \) 的卷积）


### InputLayer
**正向传播公式**：  
1. \( A = X \)

**反向传播公式**（链式法则展开）：  
1. \( \frac{\partial L}{\partial X} = \frac{\partial L}{\partial A} \cdot \frac{\partial A}{\partial X} = \frac{\partial L}{\partial A} \)（恒等传递梯度）


### 说明
**卷积层的特殊处理**：  
  - \( \star_{\text{rot}} \) 表示旋转180度后的卷积（等价于数学上的相关操作），用于计算权重梯度；  
  - \( W_{\text{rot}} \) 表示旋转后的权重，用于计算输入梯度。