DeepLearnToolbox-master NN
真没想到居然，又回来看这代码了。不过对于代码，整体还是没有搞的很清楚，只不过大概知道了意思。
- 问题：
1）对于代码，整体理解，有些细节还是不清楚。
2）
3）
4）
5）
- 创新点：
1）
2）
3）
4）
- 整体总结：
1）代码整体上，是从一个比较基础的网络开始，然后添加了各种功能。启示就是编写一个比较复杂的代码，可以先从分解为简单的功能，然后逐步添加更复杂的功能，大神都是这样编写代码的。
2）
3）
4）
5）
- 想法：
1）
2）
3）
4）
5）
先看test_example_NN.m（依次调用nnsetup.m、nntrain.m、nntest.m）
function test_example_NN
clear;close all;clc;
disp('当前正在执行的程序是：');
disp([mfilename('fullpath'),'.m']);
addpath(genpath('../data/'));
addpath(genpath('../NN/'));
addpath(genpath('../Util/'));
load mnist_uint8;
%下面的数据train_x为训练数据样本，train_y为训练数据标签。
%下面的数据test_x为测试数据样本，test_y为测试数据标签。
train_x = double(train_x) / 255;%加载数据
test_x = double(test_x) / 255;
train_y = double(train_y);
test_y = double(test_y);
% normalize
%归一化,数据（按列存储的），为0均值，标准偏差
%返回参数，train_x为规整化后的数据，mu和sigma为用来规整化的均值，和缩放比例
[train_x, mu, sigma] = zscore(train_x);
%用相同的均值和缩放比例来规整化数据
test_x = normalize(test_x, mu, sigma);
%% ex1 vanilla neural net
%vanilla 香草
rand('state',0)
nn = nnsetup([784 100 10]);%初始化网络参数
opts.numepochs = 1; % Number of full sweeps through data 整体打乱样本，重新训练的次数，每次整体训练，都重新打乱一次
opts.batchsize = 100; % Take a mean gradient step over this many samples，每批训练样本的大小
[nn, L] = nntrain(nn, train_x, train_y, opts);%训练网络
[er, bad] = nntest(nn, test_x, test_y);%测试
assert(er < 0.08, 'Too big error');
%% ex2 neural net with L2 weight decay
rand('state',0)
nn = nnsetup([784 100 10]);
nn.weightPenaltyL2 = 1e-4; % L2 weight decay
opts.numepochs = 1; % Number of full sweeps through data
opts.batchsize = 100; % Take a mean gradient step over this many samples
nn = nntrain(nn, train_x, train_y, opts);
[er, bad] = nntest(nn, test_x, test_y);
assert(er < 0.1, 'Too big error');
%% ex3 neural net with dropout
rand('state',0)
nn = nnsetup([784 100 10]);
nn.dropoutFraction = 0.5; % Dropout fraction
opts.numepochs = 1; % Number of full sweeps through data
opts.batchsize = 100; % Take a mean gradient step over this many samples
nn = nntrain(nn, train_x, train_y, opts);
[er, bad] = nntest(nn, test_x, test_y);
assert(er < 0.1, 'Too big error');
%% ex4 neural net with sigmoid activation function
%从下面简单的几步设置一个网络，发现最重要的几个函数就是nnsetup,nntrain和nntest
%所以，对于本代码中NN主要就解析这三个函数就可以
rand('state',0)
nn = nnsetup([784 100 10]);
nn.activation_function = 'sigm'; % Sigmoid activation function
nn.learningRate = 1; % Sigm require a lower learning rate
opts.numepochs = 1; % Number of full sweeps through data
opts.batchsize = 100; % Take a mean gradient step over this many samples
nn = nntrain(nn, train_x, train_y, opts);
[er, bad] = nntest(nn, test_x, test_y);
assert(er < 0.1, 'Too big error');
%% ex5 plotting functionality
rand('state',0)
nn = nnsetup([784 20 10]);
opts.numepochs = 5; % Number of full sweeps through data
nn.output = 'softmax'; % use softmax output
opts.batchsize = 1000; % Take a mean gradient step over this many samples
opts.plot = 1; % enable plotting
nn = nntrain(nn, train_x, train_y, opts);
[er, bad] = nntest(nn, test_x, test_y);
assert(er < 0.1, 'Too big error');
%% ex6 neural net with sigmoid activation and plotting of validation and training error
% split training data into training and validation data
vx = train_x(1:10000,:);
tx = train_x(10001:end,:);
vy = train_y(1:10000,:);
ty = train_y(10001:end,:);
rand('state',0)
nn = nnsetup([784 20 10]);
nn.output = 'softmax'; % use softmax output
opts.numepochs = 5; % Number of full sweeps through data
opts.batchsize = 1000; % Take a mean gradient step over this many samples
opts.plot = 1; % enable plotting
nn = nntrain(nn, tx, ty, opts, vx, vy); % nntrain takes validation set as last two arguments (optionally)
[er, bad] = nntest(nn, test_x, test_y);
assert(er < 0.1, 'Too big error');
nnsetup.m
function nn = nnsetup(architecture)
%NNSETUP creates a Feedforward Backpropagate Neural Network
% nn = nnsetup(architecture) returns an neural network structure with n=numel(architecture)
% layers, architecture being a n x 1 vector of layer sizes e.g. [784 100 10]
%首先从传入的architecture中获得这个网络的整体结构，nn.n表示这个网络有多少层
%可以参照上面的样例调用nnsetup([784 100 10])加以理解
nn.size = architecture;%nn.size完整保存模型的结构
nn.n = numel(nn.size);%模型的层数
%接下来是一大堆的参数
nn.activation_function = 'tanh_opt'; % Activation functions of hidden layers: 'sigm' (sigmoid) or 'tanh_opt' (optimal tanh).
nn.learningRate = 2; % learning rate Note: typically needs to be lower when using 'sigm' activation function and non-normalized inputs.
nn.momentum = 0.5; % Momentum
nn.scaling_learningRate = 1; % Scaling factor for the learning rate (each epoch)
nn.weightPenaltyL2 = 0; % L2 regularization
nn.nonSparsityPenalty = 0; % Non sparsity penalty
nn.sparsityTarget = 0.05; % Sparsity target
nn.inputZeroMaskedFraction = 0; % Used for Denoising AutoEncoders
nn.dropoutFraction = 0; % Dropout level (http://www.cs.toronto.edu/~hinton/absps/dropout.pdf)
nn.testing = 0; % Internal variable. nntest sets this to one.
nn.output = 'sigm'; % output unit 'sigm' (=logistic), 'softmax' and 'linear'
%对每一层的网络结构进行初始化，一共三个参数W,vW，p，其中W是主要的参数 ,vW是更新参数时的临时参数，p是所谓的sparsity
for i = 2 : nn.n
% weights and weight momentum
%w的初始化经验规则为：
%sigmoid[-4*sqrt(6)/sqrt(hiddenSize+visibleSize),4*sqrt(6)/sqrt(hiddenSize+visibleSize)]
%tanh [-sqrt(6)/sqrt(hiddenSize+visibleSize),sqrt(6)/sqrt(hiddenSize+visibleSize)]
%下面采用的是sigmoid的参数初始化经验，但是网络初始的激活函数为tanh
%rand:在（0,1）内的标准正态分布,减去0.5，就是（-0.5,0.5），然后乘以2，就是（-1,1）
%设置每层的权重，权重设置为：-1~1之间的随机数 * 4 * sqrt（6/（输入神经元数量+输出神经元数量））
nn.W{i - 1} = (rand(nn.size(i), nn.size(i - 1)+1) - 0.5) * 2 * 4 * sqrt(6 / (nn.size(i) + nn.size(i - 1)));
nn.vW{i - 1} = zeros(size(nn.W{i - 1}));
% average activations (for use with sparsity)
%从第二层开始定义p，尺寸为[1，当前层神经元个数]
nn.p{i} = zeros(1, nn.size(i));
end
end
nntrain.m（依次调用nnff.m、nnbp.m、nnapplygrads.m）
function [nn, L] = nntrain(nn, train_x, train_y, opts, val_x, val_y)
%NNTRAIN trains a neural net
% [nn, L] = nnff(nn, x, y, opts) trains the neural network nn with input x and
% output y for opts.numepochs epochs, with minibatches of size
% opts.batchsize. Returns a neural network nn with updated activations,
% errors, weights and biases, (nn.a, nn.e, nn.W, nn.b) and L, the sum
% squared error for each training minibatch.
%检测训练集的数据格式
assert(isfloat(train_x), 'train_x must be a float');
%输入参数个数
assert(nargin == 4 || nargin == 6,'number ofinput arguments must be 4 or 6')
loss.train.e = [];
loss.train.e_frac = [];
loss.val.e = [];
loss.val.e_frac = [];
opts.validation = 0;
if nargin == 6
opts.validation = 1;
end
fhandle = [];
if isfield(opts,'plot') && opts.plot == 1
fhandle = figure();
end
%m是训练样本的数量,注意在调用的时候我们设置了opt，batchsize是做batch gradient时候的大小
%这里样本存放，都是行是样本个数，列为样本维数
m = size(train_x, 1);
%提取出结构体opts中的内容
batchsize = opts.batchsize;%每个批次的尺寸
numepochs = opts.numepochs;%样本整体重新训练的次数
numbatches = m / batchsize;%计算batch的数量
%rem为取余数，就是样本个数能够被batchsize整除，限定了一下batchsize
assert(rem(numbatches, 1) == 0, 'numbatches must be a integer');
%用来存放网络的损耗函数值的
%numepochs为对于整个数据集，进行完整的计算的次数
%numbatches为对于整个数据集，要分多少个批次才能训练完一次
L = zeros(numepochs*numbatches,1);
n = 1;
for i = 1 : numepochs%迭代的代数
tic;
kk = randperm(m);%打乱样本顺序,randperm(m)生成一个乱序的1到m的数组
for l = 1 : numbatches%每批样本进行处理
%取出每批次的训练样本
batch_x = train_x(kk((l - 1) * batchsize + 1 : l * batchsize), :);
%Add noise to input (for use in denoising autoencoder)加入noise，这是denoising autoencoder需要使用到的部分 这部分请参见《Extracting and Composing Robust Features with Denoising Autoencoders》这篇论文
%具体加入的方法就是把训练样例中的一些数据调整变为0，inputZeroMaskedFraction表示了调整的比例
%nn.inputZeroMaskedFraction为在nnsetup.m中定义的是否使用Denoising AutoEncoders的标记
%非0即为添加噪声
if(nn.inputZeroMaskedFraction ~= 0)
batch_x = batch_x.*(rand(size(batch_x))>nn.inputZeroMaskedFraction);
end
%取出每批次对应的标签
batch_y = train_y(kk((l - 1) * batchsize + 1 : l * batchsize), :);
%下面三个就是前向传播、反向传播、梯度下降，构成了整个BP
nn = nnff(nn, batch_x, batch_y);%前向传播
nn = nnbp(nn);%反向传播及计算梯度
nn = nnapplygrads(nn);%梯度下降,更新模型
L(n) = nn.L;
n = n + 1;
end
t = toc;
if opts.validation == 1
loss = nneval(nn, loss, train_x, train_y, val_x, val_y);
str_perf = sprintf('; Full-batch train mse = %f, val mse = %f', loss.train.e(end), loss.val.e(end));
else
loss = nneval(nn, loss, train_x, train_y);
str_perf = sprintf('; Full-batch train err = %f', loss.train.e(end));
end
if ishandle(fhandle)
nnupdatefigures(nn, fhandle, loss, opts, i);
end
disp(['epoch ' num2str(i) '/' num2str(opts.numepochs) '. Took ' num2str(t) ' seconds' '. Mini-batch mean squared error on training set is ' num2str(mean(L((n-numbatches):(n-1)))) str_perf]);
nn.learningRate = nn.learningRate * nn.scaling_learningRate;
end
end
nnff.m
function nn = nnff(nn, x, y)
%NNFF performs a feedforward pass
% nn = nnff(nn, x, y) returns an neural network structure with updated
% layer activations, error and loss (nn.a, nn.e and nn.L)
%进行feedforward pass，其实非常简单，就是整个网络正向跑一次就可以了,当然其中有dropout和sparsity的计算
%x为训练集尺寸为[100,784],y为对应的测试集尺寸为[100,10]
n = nn.n;%网络的层数
m = size(x, 1);%样本个数
%x原来尺寸为[100,784],在每个样本增加一个维度（值为1），x原来尺寸为[100,785],
x = [ones(m,1) x];
nn.a{1} = x;%网络第一层的激活值为输入值
%feedforward pass
%根据选择的激活函数不同进行正向传播计算
%你可以回过头去看nnsetup里面的第一个参数activation_function
%sigm就是sigmoid函数,tanh_opt就是tanh的函数，这个toolbox好像有一点改变
%tanh_opt是1.7159*tanh(2/3.*A)
%网络总共只有三层，n等于3。
%第一层的激活为原始数据，所以下面的代码只计算第二层的激活值
for i = 2 : n-1
%从第二层开始，根据激活函数，逐层前向传播，这个模型整体的定义，整个网络结构都只用一种激活函数。
switch nn.activation_function
case 'sigm'
% Calculate the unit's outputs (including the bias term)
nn.a{i} = sigm(nn.a{i - 1} * nn.W{i - 1}');
case 'tanh_opt'
nn.a{i} = tanh_opt(nn.a{i - 1} * nn.W{i - 1}');
end
%dropout, dropoutFraction 是nnsetup中可以设置的一个参数
%每层都判断一下，这样每层都重新随机，这样才是dropout
%不过整个网络的系数，一次整体dropout应该也可以，而且感觉应该效率会更高
if(nn.dropoutFraction > 0)
if(nn.testing)
%由于nn.dropoutFraction定义的为丢失率，所以 1 - nn.dropoutFraction 为保留率
nn.a{i} = nn.a{i}.*(1 - nn.dropoutFraction);
else
%rand为在[0,1]之间的均值分布。
%所以这个nn.dropoutFraction为[0,1)之间的值，为丢失率
%1 - nn.dropoutFraction为保留的真实保留的概率
%用下面这个方法来使用dropout，个人感觉还是比较巧妙的
%变量nn.dropOutMask，在nn.dropoutFraction > 0的情况下，才会生成
nn.dropOutMask{i} = (rand(size(nn.a{i}))>nn.dropoutFraction);
nn.a{i} = nn.a{i}.*nn.dropOutMask{i};
end
end
%calculate running exponential activations for use with sparsity
%计算sparsity，nonSparsityPenalty 是对没达到sparsitytarget的参数的惩罚系数
if(nn.nonSparsityPenalty>0)
%没能看懂这个，为什么要对于自身乘以0.99。而且0.99+0.01=1
%这里的稀疏性都是计算的当前层的激活值？
nn.p{i} = 0.99 * nn.p{i} + 0.01 * mean(nn.a{i}, 1);
end
%Add the bias term
%把偏差放入激活函数中，原来第二层的激活值为[100,100],第一维为样本数，第二维为特征维度
%这里计算的是第二层的输出，原来激活维度为[100,100]，每个样本都添加一个偏差，尺寸变为[100,101]
nn.a{i} = [ones(m,1) nn.a{i}];
end
%前向传输，计算输出
switch nn.output
case 'sigm'
nn.a{n} = sigm(nn.a{n - 1} * nn.W{n - 1}');
case 'linear'
nn.a{n} = nn.a{n - 1} * nn.W{n - 1}';
case 'softmax'
nn.a{n} = nn.a{n - 1} * nn.W{n - 1}';
%这个为了保险起见，也减去了一个最大值
nn.a{n} = exp(bsxfun(@minus, nn.a{n}, max(nn.a{n},[],2)));
nn.a{n} = bsxfun(@rdivide, nn.a{n}, sum(nn.a{n}, 2));
end
%error and loss
%计算错误率
nn.e = y - nn.a{n};
%对应不同的激活函数，计算整体的损耗
%保存到nn.L
switch nn.output
case {'sigm', 'linear'}
nn.L = 1/2 * sum(sum(nn.e .^ 2)) / m;
case 'softmax'
nn.L = -sum(sum(y .* log(nn.a{n}))) / m;
end
end
nnbp.m
function nn = nnbp(nn)
%NNBP performs backpropagation
%nn = nnbp(nn) returns an neural network structure with updated weights
%进行back propagation的过程，过程还是比较中规中矩，和ufldl中的Neural Network讲的基本一致,值得注意的还是dropout和sparsity的部分
%代码中的d{i}就是这一层的delta值，在ufldl中有讲的
%dW{i}基本就是计算的gradient了，只是后面还要加入一些东西，进行一些修改
n = nn.n;%网络的层数
sparsityError = 0;
%针对输出不同的激活函数，计算最后一层的delta
switch nn.output
case 'sigm'
d{n} = - nn.e .* (nn.a{n} .* (1 - nn.a{n}));
case {'softmax','linear'}
d{n} = - nn.e;
end
%从后往前逐层反向传输
for i = (n - 1) : -1 : 2
%Derivative of the activation function
%对于网络中不同的激活值，反向传播计算得到的激活函数的导数不同
switch nn.activation_function
%不同激活函数的反向传播的
case 'sigm'
%计算反向传播
d_act = nn.a{i} .* (1 - nn.a{i});
case 'tanh_opt'
%tanh_opt(A)=1.7159*tanh(2/3.*A);
d_act = 1.7159 * 2/3 * (1 - 1/(1.7159)^2 * nn.a{i}.^2);
end
if(nn.nonSparsityPenalty>0)
pi = repmat(nn.p{i}, size(nn.a{i}, 1), 1);
sparsityError = [zeros(size(nn.a{i},1),1) nn.nonSparsityPenalty * (-nn.sparsityTarget ./ pi + (1 - nn.sparsityTarget) ./ (1 - pi))];
end
% Backpropagate first derivatives
if i+1==n % in this case in d{n} there is not the bias term to be removed
%这里稀疏惩罚，是加在最后一个残差上，然后逐层反向传播
d{i} = (d{i + 1} * nn.W{i} + sparsityError) .* d_act; % Bishop (5.56)
else % in this case in d{i} the bias term has to be removed
d{i} = (d{i + 1}(:,2:end) * nn.W{i} + sparsityError) .* d_act;
end
if(nn.dropoutFraction>0)
d{i} = d{i} .* [ones(size(d{i},1),1) nn.dropOutMask{i}];
end
end
%下面为计算的各个参数的偏差
for i = 1 : (n - 1)
if i+1==n
nn.dW{i} = (d{i + 1}' * nn.a{i}) / size(d{i + 1}, 1);
else
nn.dW{i} = (d{i + 1}(:,2:end)' * nn.a{i}) / size(d{i + 1}, 1);
end
end
end
nnapplygrads.m
function nn = nnapplygrads(nn)
%NNAPPLYGRADS updates weights and biases with calculated gradients
% nn = nnapplygrads(nn) returns an neural network structure with updated
% weights and biases
%nn.weightPenaltyL2 是weight decay的部分，也是nnsetup时可以设置的一个参数
%有的话就加入weight Penalty，防止过拟合，然后再根据momentum的大小调整一下，最后改变nn.W{i}即可
for i = 1 : (nn.n - 1)
if(nn.weightPenaltyL2>0)
dW = nn.dW{i} + nn.weightPenaltyL2 * [zeros(size(nn.W{i},1),1) nn.W{i}(:,2:end)];
else
dW = nn.dW{i};
end
dW = nn.learningRate * dW;
if(nn.momentum>0)
nn.vW{i} = nn.momentum*nn.vW{i} + dW;
dW = nn.vW{i};
end
nn.W{i} = nn.W{i} - dW;
end
end
nntest.m
function [er, bad] = nntest(nn, x, y)
%调用一下nnpredict，在和test的集合进行比较
labels = nnpredict(nn, x);
[~, expected] = max(y,[],2);
bad = find(labels ~= expected);
er = numel(bad) / size(x, 1);
end
浙公网安备 33010602011771号