A tiny scalar-valued autograd engine and a neural net library on top of it with PyTorch-like API

二、标量反向传播引擎Engine

1）反向传播

2）自动微分

2.2 标量Value类

Engine的核心其实就是实现了一个标量Value类，其关键就是在标量值的基础上实现基础运算和其它复杂运算（算子）的前向和反向传播（对基本运行进行了重写）。

class Value:
""" stores a single scalar value and its gradient """

def __init__(self, data, _children=(), _op=''):
self.data = data                # 标量数据

# 用于构建自动微分图的内部变量
self._backward = lambda: None   # 计算梯度的函数
self._prev = set(_children)     # 前向节点（参与该运算的Value对象集合），将用于反向传播
self._op = _op                  # 产生这个计算节点的运算类型


x = Value(1.0)
y = x + 2


out = Value(self.data + other.data, (self, other), '+')

out._backward = _backward

out = Value(1.0 + 2, (x, Value(2.0)), '+')
# 加法的链式法则，更新前向节点的梯度
def _backward():

out._backward = _backward

y = out


class Value:
""" stores a single scalar value and its gradient """

def __init__(self, data, _children=(), _op=''):
self.data = data
# internal variables used for autograd graph construction
self._backward = lambda: None
self._prev = set(_children)
self._op = _op # the op that produced this node, for graphviz / debugging / etc

# 加法
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data + other.data, (self, other), '+')

def _backward():
out._backward = _backward

return out

# 乘法
def __mul__(self, other):
other = other if isinstance(other, Value) else Value(other)
out = Value(self.data * other.data, (self, other), '*')

def _backward():
out._backward = _backward

return out

# 幂运算
def __pow__(self, other):
assert isinstance(other, (int, float)), "only supporting int/float powers for now"
out = Value(self.data**other, (self,), f'**{other}')

def _backward():
out._backward = _backward

return out

# Relu激活函数
def relu(self):
out = Value(0 if self.data < 0 else self.data, (self,), 'ReLU')

def _backward():
out._backward = _backward

return out

def backward(self):
# 先获取计算图中所有节点的拓扑顺序（也可用拓扑排序实现）
# 类似于递归式的后续遍历计算图，其遍历结果按存放于topo
topo = []
visited = set()
def build_topo(v):
if v not in visited:
for child in v._prev:
build_topo(child)
topo.append(v)  # 叶节点获取，根节点最后获取
build_topo(self)

# 上述topo实际按父节点-孩子节点顺序存放计算图的节点
# 每次获取一个节点（Value标量对象），然后调用其对应_backward()方法就能基于链式法则获取其梯度（）实现反向传播，
for v in reversed(topo):
v._backward()

def __neg__(self): # -self
return self * -1

def __radd__(self, other): # other + self
return self + other

def __sub__(self, other): # self - other
return self + (-other)

def __rsub__(self, other): # other - self
return other + (-self)

def __rmul__(self, other): # other * self
return self * other

def __truediv__(self, other): # self / other
return self * other**-1

def __rtruediv__(self, other): # other / self
return other * self**-1

def __repr__(self):


x = Value(1.0)
y = x + 2


Value类其实很类似于Pytorch中的Variable类，基于它就可构造复杂的神经网络，而不必手动的计算梯度。

三、简易网络

import random

# 类Pytorch的nn.moudle的神经网络基类
class Module:
# 参数梯度清0
for p in self.parameters():

# 返回参数（用于优化）
def parameters(self):
return []

# 神经元类（类似感知机 激活函数默认为relu）
class Neuron(Module):
# shape= [nin, 1]
def __init__(self, nin, nonlin=True):
self.w = [Value(random.uniform(-1,1)) for _ in range(nin)]
self.b = Value(0)
self.nonlin = nonlin

def __call__(self, x):
act = sum((wi*xi for wi,xi in zip(self.w, x)), self.b)
return act.relu() if self.nonlin else act

def parameters(self):
return self.w + [self.b]

def __repr__(self):
return f"{'ReLU' if self.nonlin else 'Linear'}Neuron({len(self.w)})"

# 全连接层类
class Layer(Module):
# shape = [nin, nout]
def __init__(self, nin, nout, **kwargs):
self.neurons = [Neuron(nin, **kwargs) for _ in range(nout)]

def __call__(self, x):
out = [n(x) for n in self.neurons]
return out[0] if len(out) == 1 else out

def parameters(self):
return [p for n in self.neurons for p in n.parameters()]

def __repr__(self):
return f"Layer of [{', '.join(str(n) for n in self.neurons)}]"

# 多层感知机类（或前馈神经网络FFN）
class MLP(Module):
def __init__(self, nin, nouts):
sz = [nin] + nouts
self.layers = [Layer(sz[i], sz[i+1], nonlin=i!=len(nouts)-1) for i in range(len(nouts))]

def __call__(self, x):
for layer in self.layers:
x = layer(x)
return x

def parameters(self):
return [p for layer in self.layers for p in layer.parameters()]

def __repr__(self):
return f"MLP of [{', '.join(str(layer) for layer in self.layers)}]"


参考链接

posted @ 2023-07-03 00:02  LeonYi  阅读(371)  评论(0编辑  收藏  举报