深度学习基础-基于Numpy的多层前馈神经网络（FFN）的构建和反向传播训练

class Sigmoid:
def __init__(self):
self.out = None def forward(self, x):
out = 1 / (1 + np.exp(-x))
self.out = out
return out def backward(self, dout):
dx = dout * (1.0 - self.out) * self.out
return dx

这里我们假设输入是N个样例构成的一个batch。我们可以根据计算图得出仿射变换中关于参数W和偏置B（实际计算中，会对每一个输入加相同的偏置向量，由此构成偏置矩阵。在计算其梯度时，需将每一项梯度求和）。仿射变换层的代码实现如下。

可以看到Softmax层的反向传播得到了（y1 − t1, y2 − t2, y3 − t3）这样“漂亮”的结果。由于（y1, y2, y3）是Softmax层的输出，（t1, t2, t3）是监督数据，所以（y1 − t1, y2 − t2, y3 − t3）是Softmax层的输出和真实标签的差值。神经网络的反向传播会把这个差分表示的误差传递给前面的层，这是神经网络学习中的重要性质。

在多分类中，ground truth label是一个one-hot编码，这意味着真实标签向量中仅有一维为1，而其余维都为0。假设t3为1，而t1、t2皆为0。在基于交叉熵损失优化网络参数时，误差项将以（y1, y2, y3 − 1）反向传播，以调整网络参数。 由上图可知，尽管在实际计算交叉熵损失时，只会计算真实标签向量中真正类别对应维的负对数似然，但是这个损失在反向传播中也会引导网络去调整其他层的参数。

softmax-with-loss层的代码实现如下。

import numpy as np
from collections import OrderedDictclass ThreeLayerNet:
def __init__(self, input_size, hidden_size, output_size, weight_init_std = 0.01):
print("Build Net") # 初始化权重
mid_size = hidden_size // 2
self.params = {}
self.params['W1'] = weight_init_std * np.random.randn(input_size, hidden_size)
self.params['b1'] = np.zeros(hidden_size)
self.params['W2'] = weight_init_std * np.random.randn(hidden_size, mid_size)
self.params['b2'] = np.zeros(mid_size)
self.params['W3'] = weight_init_std * np.random.randn(mid_size, output_size)
self.params['b3'] = np.zeros(output_size) # 生成层
self.layers = OrderedDict()
self.layers['Affine1'] = Affine(self.params['W1'], self.params['b1'])
self.layers['Relu1'] = Relu()
self.layers['Affine2'] = Affine(self.params['W2'], self.params['b2'])
self.layers['Relu2'] = Relu()
self.layers['Affine3'] = Affine(self.params['W3'], self.params['b3']) self.lastLayer = SoftmaxWithLoss() def predict(self, x):
for layer in self.layers.values():
x = layer.forward(x) return x def loss(self, x, t):
# x:输入数据, t:监督数据
y = self.predict(x)
return self.lastLayer.forward(y, t) def accuracy(self, x, t):
y = self.predict(x)
y = np.argmax(y, axis=1)
if t.ndim != 1 : t = np.argmax(t, axis=1) accuracy = np.sum(y == t) / float(x.shape[0])
return accuracy def gradient(self, x, t):
self.loss(x, t) # forward dout = 1 # backward
dout = self.lastLayer.backward(dout) layers = list(self.layers.values())
layers.reverse()
for layer in layers:
dout = layer.backward(dout) grads = {}
grads['W1'], grads['b1'] = self.layers['Affine1'].dW, self.layers['Affine1'].db
grads['W2'], grads['b2'] = self.layers['Affine2'].dW, self.layers['Affine2'].db
grads['W3'], grads['b3'] = self.layers['Affine3'].dW, self.layers['Affine3'].db
return grads

# coding: utf-8
try:
import urllib.request
except ImportError:
raise ImportError('You should use Python 3.x')
import os.path
import gzip
import pickle
import os
import numpy as npurl_base = 'http://yann.lecun.com/exdb/mnist/'
key_file = {
'train_img':'train-images-idx3-ubyte.gz',
'train_label':'train-labels-idx1-ubyte.gz',
'test_img':'t10k-images-idx3-ubyte.gz',
'test_label':'t10k-labels-idx1-ubyte.gz'
}dataset_dir = os.path.dirname(os.path.abspath(__file__))
save_file = dataset_dir + "/mnist.pkl"train_num = 60000
test_num = 10000
img_dim = (1, 28, 28)
img_size = 784def _download(file_name):
file_path = dataset_dir + "/" + file_name if os.path.exists(file_path):
return print("Downloading " + file_name + " ... ")
urllib.request.urlretrieve(url_base + file_name, file_path)
print("Done")def download_mnist():
for v in key_file.values():
_download(v)def _load_label(file_name):
file_path = dataset_dir + "/" + file_name print("Converting " + file_name + " to NumPy Array ...")
with gzip.open(file_path, 'rb') as f:
labels = np.frombuffer(f.read(), np.uint8, offset=8)
print("Done") return labelsdef _load_img(file_name):
file_path = dataset_dir + "/" + file_name print("Converting " + file_name + " to NumPy Array ...")
with gzip.open(file_path, 'rb') as f:
data = np.frombuffer(f.read(), np.uint8, offset=16)
data = data.reshape(-1, img_size)
print("Done") return datadef _convert_numpy():
dataset = {}
dataset['train_img'] = _load_img(key_file['train_img'])
dataset['train_label'] = _load_label(key_file['train_label'])
dataset['test_img'] = _load_img(key_file['test_img'])
dataset['test_label'] = _load_label(key_file['test_label']) return datasetdef init_mnist():
download_mnist()
dataset = _convert_numpy()
print("Creating pickle file ...")
with open(save_file, 'wb') as f:
pickle.dump(dataset, f, -1)
print("Done!")def _change_one_hot_label(X):
T = np.zeros((X.size, 10))
for idx, row in enumerate(T):
row[X[idx]] = 1 return Tdef load_mnist(normalize=True, flatten=True, one_hot_label=False):
"""读入MNIST数据集 Parameters
----------
normalize : 将图像的像素值正规化为0.0~1.0
one_hot_label :
one_hot_label为True的情况下，标签作为one-hot数组返回
one-hot数组是指[0,0,1,0,0,0,0,0,0,0]这样的数组
flatten : 是否将图像展开为一维数组 Returns
-------
(训练图像, 训练标签), (测试图像, 测试标签)
"""
if not os.path.exists(save_file):
init_mnist() with open(save_file, 'rb') as f:
dataset = pickle.load(f) if normalize:
for key in ('train_img', 'test_img'):
dataset[key] = dataset[key].astype(np.float32)
dataset[key] /= 255.0 if one_hot_label:
dataset['train_label'] = _change_one_hot_label(dataset['train_label'])
dataset['test_label'] = _change_one_hot_label(dataset['test_label']) if not flatten:
for key in ('train_img', 'test_img'):
dataset[key] = dataset[key].reshape(-1, 1, 28, 28) return (dataset['train_img'], dataset['train_label']), (dataset['test_img'], dataset['test_label']) if __name__ == '__main__':
init_mnist()

import sys, os
sys.path.append(os.pardir) # 为了导入父目录的文件而进行的设定
import numpy as np
import matplotlib.pyplot as plt
from dataset.mnist import load_mnist# 读入并划分数据
ratio = 0.1
(x_train, t_train), (x_test, t_test) = load_mnist(normalize=True, one_hot_label=True)
train_size = x_train.shape[0]
valid_size = int(train_size * ratio)
train_size = train_size - valid_size
x_valid = x_train[:valid_size]
t_valid = t_train[:valid_size]
x_train = x_train[valid_size:]
t_train = t_train[valid_size:]# 构建网络
network = ThreeLayerNet(input_size=784, hidden_size=64, output_size=10)
learning_rate = 0.01class SGD:
"""随机梯度下降法（Stochastic Gradient Descent）"""
def __init__(self, lr=0.01):
self.lr = lr def update(self, params, grads):
for key in params.keys():
params[key] -= self.lr * grads[key] # 不同参数W1, ..., Wk有相同学习率optimizer = SGD(lr=learning_rate)# 训练和验证
batch_size = 200
iters_num = 5000 # 适当设定循环的次数
iter_per_epoch = max(train_size / batch_size, 1)
train_loss_list, train_acc_list, valid_acc_list = [], [], []
for i in range(iters_num):
batch_mask = np.random.choice(train_size, batch_size)
x_batch = x_train[batch_mask]
t_batch = t_train[batch_mask]
grad = network.gradient(x_batch, t_batch) # grad = network.numerical_gradient(x_batch, t_batch)
optimizer.update(network.params, grad) # 更新参数 loss = network.loss(x_batch, t_batch)
train_loss_list.append(loss)
if i % iter_per_epoch == 0:
train_acc = network.accuracy(x_train, t_train)
valid_acc = network.accuracy(x_valid, t_valid)
train_acc_list.append(train_acc)
valid_acc_list.append(valid_acc)
print("train acc, valid acc | " + str(train_acc) + ", " + str(valid_acc))# 绘制图形 (Train, test)
markers = {'train': 'o', 'valid': 's'}
x = np.arange(len(train_acc_list))
plt.plot(x, train_acc_list, label='train acc')
plt.plot(x, valid_acc_list, label='valid acc', linestyle='--')
plt.xlabel("epochs")
plt.ylabel("accuracy")
plt.ylim(0, 1.0)
plt.legend(loc='lower right')
plt.show()# 测试
test_acc = network.accuracy(x_test, t_test)
print("test acc | " + str(test_acc))