一种C#原生实现的神经网络

矩阵运算部分来源：https://www.cnblogs.com/gpcuster/archive/2008/05/22/1204456.html
这位坛友是我用C#设计神经网络的引路人，虽然Matrix类也是简简单单的double[,]，但是至少让各种运算能进行了。提供了一个例子，我发现这个例子没有偏置，而且较难添加隐含层，于是进行了一通爆改，改成了Train(int InputNodeCount,int HiddenNodeCount,int _HiddenLayerCount,int OutputNodeCount,double LearningRate)这样较为通用的形式，可以自定义输入输出层节点数量，隐含层数和节点数量，并且加上了偏置。

代码部分实现的是8位二进制对应输出值的神经网络。因为我需要让网络能把"0 0 0 0 0 0 0 0"→“0”的问题解决出来，不得不增加偏置。午睡的时候挂机了一阵子训练了一下，在100万次的情况下也没有出现问题，应该就真的没有问题了……
在那之前真的是遇到了各种问题，比如前期训练没问题，往后的时候突然犯病，最后损失函数值成了无穷或者NaN。
目前发现会导致这个的几个原因是：隐含层节点数量不足、学习率过高、采样量占比太低，至于怎么样的超参数才是正确的，我也不好说。

先前出现的问题：

训练结果：

调用部分：
`Dictionary<int, double[]> Datas = new Dictionary<int, double[]>
{
{0,new double[]{0,0,0,0,0,0,0,0}},
{1,new double[]{0,0,0,0,0,0,0,1}},
{2,new double[]{0,0,0,0,0,0,1,0}},
{3,new double[]{0,0,0,0,0,0,1,1}},
{4,new double[]{0,0,0,0,0,1,0,0}},
{5,new double[]{0,0,0,0,0,1,0,1}},
{6,new double[]{0,0,0,0,0,1,1,0}},
{7,new double[]{0,0,0,0,0,1,1,1}},
{8,new double[]{0,0,0,0,1,0,0,0}},
{9,new double[]{0,0,0,0,1,0,0,1}},
{10,new double[]{0,0,0,0,1,0,1,0}},
{11,new double[]{0,0,0,0,1,0,1,1}},
{12,new double[]{0,0,0,0,1,1,0,0}},
{13,new double[]{0,0,0,0,1,1,0,1}},
{14,new double[]{0,0,0,0,1,1,1,0}},
{15,new double[]{0,0,0,0,1,1,1,1}},
{16,new double[]{0,0,0,1,0,0,0,0}},
{17,new double[]{0,0,0,1,0,0,0,1}},
{18,new double[]{0,0,0,1,0,0,1,0}},
{19,new double[]{0,0,0,1,0,0,1,1}},
{20,new double[]{0,0,0,1,0,1,0,0}},
{21,new double[]{0,0,0,1,0,1,0,1}},
{22,new double[]{0,0,0,1,0,1,1,0}},
{23,new double[]{0,0,0,1,0,1,1,1}},
{24,new double[]{0,0,0,1,1,0,0,0}},
{25,new double[]{0,0,0,1,1,0,0,1}},
{26,new double[]{0,0,0,1,1,0,1,0}},
{27,new double[]{0,0,0,1,1,0,1,1}},
{28,new double[]{0,0,0,1,1,1,0,0}},
{29,new double[]{0,0,0,1,1,1,0,1}},
{30,new double[]{0,0,0,1,1,1,1,0}},
{31,new double[]{0,0,0,1,1,1,1,1}},
{32,new double[]{0,0,1,0,0,0,0,0}},
{33,new double[]{0,0,1,0,0,0,0,1}},
{34,new double[]{0,0,1,0,0,0,1,0}},
{35,new double[]{0,0,1,0,0,0,1,1}},
{36,new double[]{0,0,1,0,0,1,0,0}},
{37,new double[]{0,0,1,0,0,1,0,1}},
{38,new double[]{0,0,1,0,0,1,1,0}},
{39,new double[]{0,0,1,0,0,1,1,1}},
{40,new double[]{0,0,1,0,1,0,0,0}},
{41,new double[]{0,0,1,0,1,0,0,1}},
{42,new double[]{0,0,1,0,1,0,1,0}},
{43,new double[]{0,0,1,0,1,0,1,1}},
{44,new double[]{0,0,1,0,1,1,0,0}},
{45,new double[]{0,0,1,0,1,1,0,1}},
{46,new double[]{0,0,1,0,1,1,1,0}},
{47,new double[]{0,0,1,0,1,1,1,1}},
{48,new double[]{0,0,1,1,0,0,0,0}},
{49,new double[]{0,0,1,1,0,0,0,1}},
{50,new double[]{0,0,1,1,0,0,1,0}},
{51,new double[]{0,0,1,1,0,0,1,1}},
{52,new double[]{0,0,1,1,0,1,0,0}},
{53,new double[]{0,0,1,1,0,1,0,1}},
{54,new double[]{0,0,1,1,0,1,1,0}},
{55,new double[]{0,0,1,1,0,1,1,1}},
{56,new double[]{0,0,1,1,1,0,0,0}},
{57,new double[]{0,0,1,1,1,0,0,1}},
{58,new double[]{0,0,1,1,1,0,1,0}},
{59,new double[]{0,0,1,1,1,0,1,1}},
{60,new double[]{0,0,1,1,1,1,0,0}},
{61,new double[]{0,0,1,1,1,1,0,1}},
{62,new double[]{0,0,1,1,1,1,1,0}},
{63,new double[]{0,0,1,1,1,1,1,1}},
{64,new double[]{0,1,0,0,0,0,0,0}},
{65,new double[]{0,1,0,0,0,0,0,1}},
{66,new double[]{0,1,0,0,0,0,1,0}},
{67,new double[]{0,1,0,0,0,0,1,1}},
{68,new double[]{0,1,0,0,0,1,0,0}},
{69,new double[]{0,1,0,0,0,1,0,1}},
{70,new double[]{0,1,0,0,0,1,1,0}},
{71,new double[]{0,1,0,0,0,1,1,1}},
{72,new double[]{0,1,0,0,1,0,0,0}},
{73,new double[]{0,1,0,0,1,0,0,1}},
{74,new double[]{0,1,0,0,1,0,1,0}},
{75,new double[]{0,1,0,0,1,0,1,1}},
{76,new double[]{0,1,0,0,1,1,0,0}},
{77,new double[]{0,1,0,0,1,1,0,1}},
{78,new double[]{0,1,0,0,1,1,1,0}},
{79,new double[]{0,1,0,0,1,1,1,1}},
{80,new double[]{0,1,0,1,0,0,0,0}},
{81,new double[]{0,1,0,1,0,0,0,1}},
{82,new double[]{0,1,0,1,0,0,1,0}},
{83,new double[]{0,1,0,1,0,0,1,1}},
{84,new double[]{0,1,0,1,0,1,0,0}},
{85,new double[]{0,1,0,1,0,1,0,1}},
{86,new double[]{0,1,0,1,0,1,1,0}},
{87,new double[]{0,1,0,1,0,1,1,1}},
{88,new double[]{0,1,0,1,1,0,0,0}},
{89,new double[]{0,1,0,1,1,0,0,1}},
{90,new double[]{0,1,0,1,1,0,1,0}},
{91,new double[]{0,1,0,1,1,0,1,1}},
{92,new double[]{0,1,0,1,1,1,0,0}},
{93,new double[]{0,1,0,1,1,1,0,1}},
{94,new double[]{0,1,0,1,1,1,1,0}},
{95,new double[]{0,1,0,1,1,1,1,1}},
{96,new double[]{0,1,1,0,0,0,0,0}},
{97,new double[]{0,1,1,0,0,0,0,1}},
{98,new double[]{0,1,1,0,0,0,1,0}},
{99,new double[]{0,1,1,0,0,0,1,1}},
{100,new double[]{0,1,1,0,0,1,0,0}},
{101,new double[]{0,1,1,0,0,1,0,1}},
{102,new double[]{0,1,1,0,0,1,1,0}},
{103,new double[]{0,1,1,0,0,1,1,1}},
{104,new double[]{0,1,1,0,1,0,0,0}},
{105,new double[]{0,1,1,0,1,0,0,1}},
{106,new double[]{0,1,1,0,1,0,1,0}},
{107,new double[]{0,1,1,0,1,0,1,1}},
{108,new double[]{0,1,1,0,1,1,0,0}},
{109,new double[]{0,1,1,0,1,1,0,1}},
{110,new double[]{0,1,1,0,1,1,1,0}},
{111,new double[]{0,1,1,0,1,1,1,1}},
{112,new double[]{0,1,1,1,0,0,0,0}},
{113,new double[]{0,1,1,1,0,0,0,1}},
{114,new double[]{0,1,1,1,0,0,1,0}},
{115,new double[]{0,1,1,1,0,0,1,1}},
{116,new double[]{0,1,1,1,0,1,0,0}},
{117,new double[]{0,1,1,1,0,1,0,1}},
{118,new double[]{0,1,1,1,0,1,1,0}},
{119,new double[]{0,1,1,1,0,1,1,1}},
{120,new double[]{0,1,1,1,1,0,0,0}},
{121,new double[]{0,1,1,1,1,0,0,1}},
{122,new double[]{0,1,1,1,1,0,1,0}},
{123,new double[]{0,1,1,1,1,0,1,1}},
{124,new double[]{0,1,1,1,1,1,0,0}},
{125,new double[]{0,1,1,1,1,1,0,1}},
{126,new double[]{0,1,1,1,1,1,1,0}},
{127,new double[]{0,1,1,1,1,1,1,1}},
{128,new double[]{1,0,0,0,0,0,0,0}},
{129,new double[]{1,0,0,0,0,0,0,1}},
{130,new double[]{1,0,0,0,0,0,1,0}},
{131,new double[]{1,0,0,0,0,0,1,1}},
{132,new double[]{1,0,0,0,0,1,0,0}},
{133,new double[]{1,0,0,0,0,1,0,1}},
{134,new double[]{1,0,0,0,0,1,1,0}},
{135,new double[]{1,0,0,0,0,1,1,1}},
{136,new double[]{1,0,0,0,1,0,0,0}},
{137,new double[]{1,0,0,0,1,0,0,1}},
{138,new double[]{1,0,0,0,1,0,1,0}},
{139,new double[]{1,0,0,0,1,0,1,1}},
{140,new double[]{1,0,0,0,1,1,0,0}},
{141,new double[]{1,0,0,0,1,1,0,1}},
{142,new double[]{1,0,0,0,1,1,1,0}},
{143,new double[]{1,0,0,0,1,1,1,1}},
{144,new double[]{1,0,0,1,0,0,0,0}},
{145,new double[]{1,0,0,1,0,0,0,1}},
{146,new double[]{1,0,0,1,0,0,1,0}},
{147,new double[]{1,0,0,1,0,0,1,1}},
{148,new double[]{1,0,0,1,0,1,0,0}},
{149,new double[]{1,0,0,1,0,1,0,1}},
{150,new double[]{1,0,0,1,0,1,1,0}},
{151,new double[]{1,0,0,1,0,1,1,1}},
{152,new double[]{1,0,0,1,1,0,0,0}},
{153,new double[]{1,0,0,1,1,0,0,1}},
{154,new double[]{1,0,0,1,1,0,1,0}},
{155,new double[]{1,0,0,1,1,0,1,1}},
{156,new double[]{1,0,0,1,1,1,0,0}},
{157,new double[]{1,0,0,1,1,1,0,1}},
{158,new double[]{1,0,0,1,1,1,1,0}},
{159,new double[]{1,0,0,1,1,1,1,1}},
{160,new double[]{1,0,1,0,0,0,0,0}},
{161,new double[]{1,0,1,0,0,0,0,1}},
{162,new double[]{1,0,1,0,0,0,1,0}},
{163,new double[]{1,0,1,0,0,0,1,1}},
{164,new double[]{1,0,1,0,0,1,0,0}},
{165,new double[]{1,0,1,0,0,1,0,1}},
{166,new double[]{1,0,1,0,0,1,1,0}},
{167,new double[]{1,0,1,0,0,1,1,1}},
{168,new double[]{1,0,1,0,1,0,0,0}},
{169,new double[]{1,0,1,0,1,0,0,1}},
{170,new double[]{1,0,1,0,1,0,1,0}},
{171,new double[]{1,0,1,0,1,0,1,1}},
{172,new double[]{1,0,1,0,1,1,0,0}},
{173,new double[]{1,0,1,0,1,1,0,1}},
{174,new double[]{1,0,1,0,1,1,1,0}},
{175,new double[]{1,0,1,0,1,1,1,1}},
{176,new double[]{1,0,1,1,0,0,0,0}},
{177,new double[]{1,0,1,1,0,0,0,1}},
{178,new double[]{1,0,1,1,0,0,1,0}},
{179,new double[]{1,0,1,1,0,0,1,1}},
{180,new double[]{1,0,1,1,0,1,0,0}},
{181,new double[]{1,0,1,1,0,1,0,1}},
{182,new double[]{1,0,1,1,0,1,1,0}},
{183,new double[]{1,0,1,1,0,1,1,1}},
{184,new double[]{1,0,1,1,1,0,0,0}},
{185,new double[]{1,0,1,1,1,0,0,1}},
{186,new double[]{1,0,1,1,1,0,1,0}},
{187,new double[]{1,0,1,1,1,0,1,1}},
{188,new double[]{1,0,1,1,1,1,0,0}},
{189,new double[]{1,0,1,1,1,1,0,1}},
{190,new double[]{1,0,1,1,1,1,1,0}},
{191,new double[]{1,0,1,1,1,1,1,1}},
{192,new double[]{1,1,0,0,0,0,0,0}},
{193,new double[]{1,1,0,0,0,0,0,1}},
{194,new double[]{1,1,0,0,0,0,1,0}},
{195,new double[]{1,1,0,0,0,0,1,1}},
{196,new double[]{1,1,0,0,0,1,0,0}},
{197,new double[]{1,1,0,0,0,1,0,1}},
{198,new double[]{1,1,0,0,0,1,1,0}},
{199,new double[]{1,1,0,0,0,1,1,1}},
{200,new double[]{1,1,0,0,1,0,0,0}},
{201,new double[]{1,1,0,0,1,0,0,1}},
{202,new double[]{1,1,0,0,1,0,1,0}},
{203,new double[]{1,1,0,0,1,0,1,1}},
{204,new double[]{1,1,0,0,1,1,0,0}},
{205,new double[]{1,1,0,0,1,1,0,1}},
{206,new double[]{1,1,0,0,1,1,1,0}},
{207,new double[]{1,1,0,0,1,1,1,1}},
{208,new double[]{1,1,0,1,0,0,0,0}},
{209,new double[]{1,1,0,1,0,0,0,1}},
{210,new double[]{1,1,0,1,0,0,1,0}},
{211,new double[]{1,1,0,1,0,0,1,1}},
{212,new double[]{1,1,0,1,0,1,0,0}},
{213,new double[]{1,1,0,1,0,1,0,1}},
{214,new double[]{1,1,0,1,0,1,1,0}},
{215,new double[]{1,1,0,1,0,1,1,1}},
{216,new double[]{1,1,0,1,1,0,0,0}},
{217,new double[]{1,1,0,1,1,0,0,1}},
{218,new double[]{1,1,0,1,1,0,1,0}},
{219,new double[]{1,1,0,1,1,0,1,1}},
{220,new double[]{1,1,0,1,1,1,0,0}},
{221,new double[]{1,1,0,1,1,1,0,1}},
{222,new double[]{1,1,0,1,1,1,1,0}},
{223,new double[]{1,1,0,1,1,1,1,1}},
{224,new double[]{1,1,1,0,0,0,0,0}},
{225,new double[]{1,1,1,0,0,0,0,1}},
{226,new double[]{1,1,1,0,0,0,1,0}},
{227,new double[]{1,1,1,0,0,0,1,1}},
{228,new double[]{1,1,1,0,0,1,0,0}},
{229,new double[]{1,1,1,0,0,1,0,1}},
{230,new double[]{1,1,1,0,0,1,1,0}},
{231,new double[]{1,1,1,0,0,1,1,1}},
{232,new double[]{1,1,1,0,1,0,0,0}},
{233,new double[]{1,1,1,0,1,0,0,1}},
{234,new double[]{1,1,1,0,1,0,1,0}},
{235,new double[]{1,1,1,0,1,0,1,1}},
{236,new double[]{1,1,1,0,1,1,0,0}},
{237,new double[]{1,1,1,0,1,1,0,1}},
{238,new double[]{1,1,1,0,1,1,1,0}},
{239,new double[]{1,1,1,0,1,1,1,1}},
{240,new double[]{1,1,1,1,0,0,0,0}},
{241,new double[]{1,1,1,1,0,0,0,1}},
{242,new double[]{1,1,1,1,0,0,1,0}},
{243,new double[]{1,1,1,1,0,0,1,1}},
{244,new double[]{1,1,1,1,0,1,0,0}},
{245,new double[]{1,1,1,1,0,1,0,1}},
{246,new double[]{1,1,1,1,0,1,1,0}},
{247,new double[]{1,1,1,1,0,1,1,1}},
{248,new double[]{1,1,1,1,1,0,0,0}},
{249,new double[]{1,1,1,1,1,0,0,1}},
{250,new double[]{1,1,1,1,1,0,1,0}},
{251,new double[]{1,1,1,1,1,0,1,1}},
{252,new double[]{1,1,1,1,1,1,0,0}},
{253,new double[]{1,1,1,1,1,1,0,1}},
{254,new double[]{1,1,1,1,1,1,1,0}},
{255,new double[]{1,1,1,1,1,1,1,1}},
};
var LastTrain = new Train(Datas[0].Length, Datas[0].Length3/2 , 1, Datas.Count, (double)LearningRate.Value);
//从5个输入起，隐含层节点数改为了Datas[0].Length3/2。再用3个隐含层节点训练就会爆炸了。
//7个输入的时候10个节点也不够了？改成输入节点x2？23608失败 24351失败 23497失败。
//改成x3 1层 23678失败 22847失败可能不是隐含层节点问题？
//改成x1 2层 648失败
//改成x2 2层 1518失败 1368失败 1429失败
//改成x5 1层 23435失败确定和隐含层节点数没关系了
//经过研究发现是采样数量不够。对于128的数据集，20可以通过。暂定为log2(n)*3虽然不会指数爆炸但是感觉将来会有暗病，其实目的是按比例增加采样数量。

//隐含层节点暂定为输入层节点的1.5倍
//采样数暂定为输入数量的1/8
//学习率维持0.1
Random r = new Random();
double Min = 100;
for (int i = 0; i < TrainTimes.Value; i++)//训练次数
{
double Loss = 0;
int index = 0;
for (int j = 0; j < SliceSize.Value; j++)//从总训练集中取的采样数量，用随机梯度下降。
{
//Input W1 z1 b1 o1(Input2) W2 z2
//Layer是Input到计算出o1给下一层。最后一个隐含层只会算出来一个z2，需要经过softmax与损失函数计算。
index = r.Next() % Datas.Count;
LastTrain.LayerHead.Calculate(new Train.Matrix(Datas[index]));
var result = LastTrain.LayerTail.FirstBP(index);
Loss += result;
}
Loss /= (int)SliceSize.Value;
if (Min > Loss)
{
Min = Loss;
//LastTrain.Save("F:\value.bin");//存出来最优值用于实际运用
}
if (double.IsNaN(Loss))
{
Log.AppendText($"第{i}次训练差值：{Loss}\r\n");
break;
}
if (i % 1000 == 0) Log.AppendText($"第{i}次训练差值：{Loss}\r\n");
LastTrain.LayerTail.Apply(SliceSize.Value);//应用修改值。之前没有算平均，相当于学习率会被采样数量增幅导致数字问题从而训练爆炸。
}
`

下边的代码里有一些算式相关的，如果您也想计算，我把图放出来，可以对照着看一下。

Train对象：
`internal class Train
{
public class Matrix
{
public double[,] Mat;
private long _m, _n;
public long M
{
get
{
return _m;
}
private set
{
_m = value;
}
}
public long N
{
get
{
return _n;
}
private set
{
_n = value;
}
}

class HERandom
{
private static Random random = new Random();
// 生成均值为0，标准差为1的正态分布随机数
public static double NextGaussian(out double num2)
{
double u = random.NextDouble();
double v = random.NextDouble();

while (u <= 0 || v <= 0)
{
u = random.NextDouble();
v = random.NextDouble();
}

double num = Math.Sqrt(-2.0 * Math.Log(u)) * Math.Cos(2.0 * Math.PI * v);
num2 = Math.Sqrt(-2.0 * Math.Log(u)) * Math.Sin(2.0 * Math.PI * v);
return num;
}

// 生成均值为mean，标准差为stdDev的正态分布随机数
//public static double NextGaussian(double mean, double stdDev)
//{
// return mean + stdDev * NextGaussian();
//}
}
///

/// 当前用Kaiming初始化优化ReLU
///

///
///
///
public Matrix(long m, long n, bool isRandValue = false)
{
_m = m;
_n = n;
double std = Math.Sqrt(2d / m);
Mat = new double[m, n];
double temp = double.NaN;
for (int i = 0; i < m; i++)
for (int j = 0; j < n; j++)
if (isRandValue)
{
if ((i + j) % 2 == 0) Mat[i, j] = HERandom.NextGaussian(out temp) * std;//偶数生成新的，
else Mat[i, j] = temp * std;//奇数用上次生成多的
}
else Mat[i, j] = 0;
}
public Matrix(double[,] m)
{
_m = m.GetLongLength(0);
_n = m.GetLongLength(1);
Mat = m;
}
public Matrix(double[] m)
{
_m = 1;
_n = m.Length;
Mat = new double[_m,_n];
for (int i = 0; i < _n; i++) Mat[0, i] = m[i];
}
public Matrix(BinaryReader br)
{
M = br.ReadInt64();
N = br.ReadInt64();
Mat = new double[M, N];
for (int i = 0; i < M; i++)
for (int j = 0; j < N; j++) Mat[i, j] = br.ReadDouble();
}

public static Matrix operator +(Matrix M1, Matrix M2)
{
if (M1.M != M2.M
|| M1.N != M1.N)
throw new Exception("矩阵不符合运算条件，2个矩阵必须完全一样的行和列");

Matrix result = new Matrix(M1.M, M1.N);
for (int i = 0; i < M1.M; i++)
for (int j = 0; j < M1.N; j++)
{
result.Mat[i, j] = M1.Mat[i, j] + M2.Mat[i, j];
}
return result;
}
public static Matrix operator &(Matrix M1, Matrix M2)
{
if (M2.M!=1
|| M1.N != M1.N)
throw new Exception("矩阵不符合运算条件。M2必须为单行矩阵，2个矩阵必须完全一样的列");

Matrix result = new Matrix(M1.M, M1.N);
for (int i = 0; i < M1.M; i++)
for (int j = 0; j < M1.N; j++)
{
result.Mat[i, j] = M1.Mat[i, j] + M2.Mat[0, j];
}
return result;
}

public static Matrix operator -(Matrix M1, Matrix M2)
{
if (M1.M != M2.M
|| M1.N != M1.N)
throw new Exception("矩阵不符合运算条件，2个矩阵必须完全一样的行和列");

Matrix result = new Matrix(M1.M, M1.N);
for (int i = 0; i < M1.M; i++)
for (int j = 0; j < M1.N; j++)
{
result.Mat[i, j] = M1.Mat[i, j] - M2.Mat[i, j];
}
return result;
}
public static Matrix operator ^(Matrix M1, Matrix M2)
{
if (M1.M != M2.M
|| M1.N != M1.N)
throw new Exception("矩阵不符合运算条件，2个矩阵必须完全一样的行和列");

Matrix result = new Matrix(M1.M, M1.N);
for (int i = 0; i < M1.M; i++)
for (int j = 0; j < M1.N; j++)
{
result.Mat[i, j] = M1.Mat[i, j] * M2.Mat[i, j];
}
return result;
}

public static Matrix operator *(Matrix M1, Matrix M2)
{
long m = M1.Mat.GetLongLength(0);
long jW = M1.Mat.GetLongLength(1);

long iH = M2.Mat.GetLongLength(0);
long n = M2.Mat.GetLongLength(1);

if (jW != iH)
throw new Exception("矩阵不符合运算条件，W的行不等于H的列");
Matrix result = new Matrix(m, n);

for (int i = 0; i < m; i++)//W的行数
{
for (int j = 0; j < n; j++)//H的列数
{
for (int k = 0; k < jW; k++) result.Mat[i, j] += M2.Mat[k, j] * M1.Mat[i, k];
if (double.IsInfinity(result.Mat[i, j])) throw new Exception($"矩阵乘法运算超限:{i}{j}");
}
}

return result;
}
public static Matrix operator *(Matrix M1, double ratio)
{
long m = M1.Mat.GetLongLength(0);
long n = M1.Mat.GetLongLength(1);
Matrix result = new Matrix(m, n);
for (int i = 0; i < m; i++)
for (int j = 0; j < n; j++)
result.Mat[i, j] = M1.Mat[i, j] * ratio;
return result;
}
///

/// 非线性激活函数
///

///
public Matrix Nonlin()
{
Matrix result = new Matrix(M, N);
for (int i = 0; i < M; i++)
for (int j = 0; j < N; j++)
result.Mat[i, j] = ReLU(Mat[i, j]);
return result;
}
public Matrix Derivative()
{
Matrix result = new Matrix(M, N);
for (int i = 0; i < M; i++)
for (int j = 0; j < N; j++)
result.Mat[i, j] = ReLU(Mat[i, j],true);
return result;
}
public Matrix Softmax()//有几种优化思路比如safe或者online……但是那些的导数我都拿不准。
{
Matrix result = new Matrix(M, N);
for (int i = 0; i < M; i++)
{
double Sum = 0;
//double Max = Mat[i, 0];
//for (int j = 1; j < N; j++) if (Max < Mat[i, j]) Max = Mat[i, j];
for (int j = 0; j < N; j++)
{
result.Mat[i, j] = Math.Exp(Mat[i, j]);//改成safe softmax并不能阻止问题的出现（- Max）。问题的根源是数字太大了。一批从-10000~10000的数字，所有都减去Max后，e^-5000次方计算结果是0，从而产生了无穷大的损失函数。
//数字为什么会太大？一个猜想是某一个样例因为太长时间没被选择到，所以应该提高单批次的采样数量！
Sum += result.Mat[i, j];
}
for (int j = 0; j < N; j++)
result.Mat[i, j] = result.Mat[i, j] / Sum;
}
return result;
}
public Matrix T
{
get
{
Matrix result = new Matrix(N, M);

//新矩阵生成规则： b[i,j]=a[j,i]
for (int i = 0; i < N; i++)
for (int j = 0; j < M; j++)
result.Mat[i, j] = this.Mat[j, i];
return result;
}
}
public void Clear()
{
for (int i = 0; i < _m; i++)
for (int j = 0; j < _n; j++)
Mat[i, j] = 0;
}
public override string ToString()
{
StringBuilder sbd = new StringBuilder();
for (int i = 0; i < this.M; i++)
{
for (int j = 0; j < this.N; j++)
{
sbd.Append(Mat[i, j].ToString("N10"));
sbd.Append(",");
}
sbd.AppendLine();
}
return sbd.ToString();
}
public void Save(BinaryWriter bw)
{
bw.Write(M);
bw.Write(N);
for (int i = 0; i < M; i++)
for (int j = 0; j < N; j++) bw.Write(Mat[i, j]);
}
}

public static double Sigmoid(double x,bool Derivative=false)
{
if(Derivative) return x * (1 - x);//3*
else return 1.0 / (1.0 + Math.Exp(-x));//-x3
//有一种3的处理方法
}
public static double ReLU(double x, bool Derivative = false)
{
if (x > 0)
{
if (Derivative) return 1;
else return x;
}
else
{
if (Derivative) return 0.01;
else return 0.01 * x;
}
}
public static double Step(double x, bool Derivative = false)
{
if (Derivative) return 0;//不连续的函数哪来的导数
else return x >= 0 ? 1 : 0;
}

public class Layer
{
public static double LearningRate=0.01;
public Matrix Inputs;
///

/// 未经过ReLU的输出值
///

public Matrix Outputs;
public Matrix Weights;
public Matrix _weights;//改变量
public Matrix Bias;//不然没法处理0 0 0→0的问题
public Matrix _bias;//改变量
public Layer Next;
public Layer Last;
public Layer(long InputCount, long OutputCount, Layer last = null, bool isRandValue = true)
{
//if (Last == null) Bias = new Matrix(InputCount, 1, false);
//else Bias = new Matrix(InputCount, 1, true);
Last = last;
Weights = new Matrix(InputCount, OutputCount, isRandValue);
_weights = new Matrix(InputCount, OutputCount, false);
Bias = new Matrix(1, OutputCount, isRandValue);//初始化偏置
_bias = new Matrix(1, OutputCount, false);
}
public Layer(BinaryReader br)
{
Weights = new Matrix(br);
Bias = new Matrix(br);
}
public void Calculate(Matrix Input)
{
Inputs = Input;//这是一个m行n列的矩阵，每行是一组训练数据。（并不是，这是一个1行n列的矩阵，不需要一次把一堆输入塞进去）
Outputs = Inputs * Weights;//1行Weights.N列的向量
Outputs = Outputs & Bias;
Next?.Calculate(Outputs.Nonlin());//用ReLU处理过后的给下一层处理
}
///

/// 最后一层需要用softmax，所以单独取出来。目的是获取到损失函数对未压缩的数值的导数 dLoss/dZ。
///

/// 期望输出值
public double FirstBP(int Aim)
{
var A = Outputs.Softmax();
var Loss = -Math.Log(A.Mat[0, Aim]);

double[] dz2 = new double[A.N];//dLoss/da2X * da2X/dz2X 最终得到X个偏导数。

for (int j = 0; j < A.N; j++)//输出节点个数。
{
dz2[j] = A.Mat[0, j];//这个训练数据的目标输出是Aim，jAim时，Loss对第Aim个Output的偏导是Output[j]-1。对其他的则是Output[j]
if (Aim == j) dz2[j]--;
//当Aim=j时，dz2[j]= dLoss/da2j * da2j/dz2j = -1/a21* a21(1-a21)=a21-1 也就是a2Aim -1
//当Aim!=j时，dz2[j]=dLoss/da2Aim * da2Aim/dz2j = -1/a21 * -a21a2j =a2j
}
BP(dz2);//现在算出来的是对z的导数，传给自己重新做统一的计算吧。
return Loss;
//a21=e^z21/(ez21+e^z22+ez23+e^z24)。
//Loss=-ln(a21)（如果j=1。也就是Loss只对a21有导数，不过a21对z21 z22 z23 z24都能求偏导）
//dLoss/da21=-1/a21。
//da21/dz21=d(e^z21/(ez21+e^z22+ez23))/dz21= a21 + ez21(-e^z21/(ez21+e^z22+ez23)^2)=a21(1-a21) ……居然让我自己推导出来了（算错3次
//da21/dz22=d(e^z21/(ez21+e^z22+ez23))/dz22 ： (uv)'=u'v+uv'。de^{z21/dz22=0，所以第一项为0。第二项是e}z21(-1/(e^z21+ez22+e^z23)2)z22=-a21a22
//dz2是Loss对A21,A22,A23,A24的求导。其中只有A2j的求导是A2j-1，其余的都是A2i。
//Z2=∑a1w2+b2，Z21=a11w31+a12w41+a13w51+b21 A21的值假设为A21,j0，dLoss/dA21=A21-1。
//dLoss/db21= dLoss/dA21 * dA21/dZ21 * dZ21/db21。
//dA21/dZ21是对ReLU求导，根据dZ21的符号决定是1还是0.01。 dZ21/db21就是简单的1。
}
///

/// 通过对Outputs的导数计算这一层的权重和偏置应该如何改变。如果还有上一层，就提供对上一层的偏导供继续传播使用。
///

/// 损失函数对当前层的Outputs的导数
public void BP(double[] d)//提供这一层z的导数，使用它往前推断
{
//需要更新的是权重表和偏置表
for(int i=0;i<Weights.N;i++)
{
//和zi有关的是bi和wXi
_bias.Mat[0,i] += d[i] * LearningRate;//dLoss/dz21 * dz21/db21=d[i]* d(a11w31+a12w41+a13w51+b21)/db21=d[i]1

for (int j = 0; j < Weights.M; j++)
{
_weights.Mat[j,i] += d[i] Inputs.Mat[0,j]LearningRate;//dLoss/dz31 * dz21/dw31=d[i]* d(a11w31+a12w41+a13w51+b21)/dw31= d[i]a11
//确实出现了下标需要反过来写的问题。我写的w31 w41 w51是节点3 4 5连到下一层第一个节点的意思，然而在这里1是i。
}
}
if (Last == null) return;//下边都是算给上一层用的东西的，没必要
double[] Pass = new double[Inputs.N];
//计算对a11的偏导 a11通过w31 w32 w33 w34影响了z21 z22 z23 z24，所以这四部分的影响要加起来……对吧
for(int i = 0; i < Pass.Length; i++)
{
//double Sum = 0;
//for (int j = 0; j < d.Length; j++) Sum += Weights.Mat[i, j];
//Sum /= d.Length;
for (int j = 0; j < d.Length; j++) {
Pass[i] += d[j] * Weights.Mat[i,j];//Sum 这个Sum不除也罢。
//dLoss/da11=dLoss/dz21 * dz21/da11 + dLoss/dz22 * dz22/da11 + dLoss/dz23 * dz23/da11 + dLoss/dz24 * dz24/da11
//dz21/da11=d(a11w31+a12w41+a13w51+b21)/da11= w31
}
//Pass[i] /= d.Length;//（要平均吗？看别人的代码没有除法，大概不用）
}
//计算ReLU导数
for(int i = 0; i < Pass.Length; i++)
{
Pass[i]=ReLU(Pass[i], true);//if (Inputs.Mat[0, i] < 0) Pass[i] = 0.01;//ReLU的导数，如果输出结果小于零，导数就是0.01。否则导数是1，也就不用乘了。
}
Last?.BP(Pass);
//反向传播……https://zhuanlan.zhihu.com/p/25723112 似乎很好算，但是如果在意计算过程的话就寄了。
}
///

/// 需要对最后一层使用。
///

public void Apply(decimal times)
{
Weights -= _weights(1/(double)times);//因为这里边大部分都是正数，所以越来越小了……但是也绝对不是+=
Bias -= _bias * (1 / (double)times);
_weights.Clear();
_bias.Clear();
Last?.Apply(times);
}
}
///

/// 链表头
///

public Layer LayerHead;
///

/// 链表尾
///

public Layer LayerTail;
int HiddenLayerCount;
public Train(int InputNodeCount,int HiddenNodeCount,int _HiddenLayerCount,int OutputNodeCount,double LearningRate)
{
if (_HiddenLayerCount < 1) throw new Exception("至少要有一层隐含层");
HiddenLayerCount = _HiddenLayerCount;
LayerHead = new Layer(InputNodeCount,HiddenNodeCount);
Layer pointer = LayerHead;
for(int i = 0; i < _HiddenLayerCount; i++)
{
if (i == _HiddenLayerCount - 1) pointer.Next = new Layer(HiddenNodeCount, OutputNodeCount, pointer);
else pointer.Next = new Layer(HiddenNodeCount, HiddenNodeCount, pointer);
pointer = pointer.Next;
}
LayerTail = pointer;
Layer.LearningRate = LearningRate;
}
///

/// 提供多组输入Input，以及按顺序每组的输出Result
///

public void TrainData(double[] Input, int Result)
{
LayerHead.Calculate(new Matrix(Input));
//LayerTail.FirstBP(Result);
}

///

/// 保存模型的参数
///

public void Save(string FilePath)
{
using (FileStream fs = new FileStream(FilePath, FileMode.Create))
using (BinaryWriter bw = new BinaryWriter(fs))
{
bw.Write(HiddenLayerCount);
Layer p = LayerHead;
while (p != null)
{
p.Weights.Save(bw);
p.Bias.Save(bw);

p = p.Next;
}
}
}
public Train(string FilePath)
{
using(FileStream fs=new FileStream(FilePath,FileMode.Open))
using(BinaryReader br=new BinaryReader(fs))
{
HiddenLayerCount = br.ReadInt32();
LayerHead = new Layer(br);
Layer p = LayerHead;
for(int i = 0; i < HiddenLayerCount; i++)
{
Layer l = new Layer(br);
p.Next = l;
l.Last = p;
p = p.Next;
}
LayerTail = p;
}
}
}
`

以前C++学链表，总得自己把数据结构写出来一遍才认为学会了。神经网络虽然可以直接照用网上的python代码拿去跑，但是终究自己不知道它的原理，不知道的话就更没办法优化了。为了用自己熟悉的C#实现神经网络，经历了一两个月的研究，期间还被一些大语言模型搜索引擎骗过=_=' 不过终究是研究出来了，甚至了解了一下已经忘完了的导数计算规则，发现自己能推导出来网上说的损失函数对softmax甚至axx求导的数值了。把公式拆开，一项一项计算在我看来是搞清楚神经网络的重要一步，在那之前对自己来说就完全是一个黑箱。
坛友提供的矩阵运算方式没有用什么魔法，是很朴实的CPU计算。后边需要研究怎么用cuda加速这一过程了，想要实现一个属于自己的神经网络。

posted @ 2024-12-02 15:11 星谷望月阅读(66) 评论(0) 收藏举报

刷新页面返回顶部

hoshitani

一种C#原生实现的神经网络

公告