# 2 前向自动微分

$\begin{gathered} {\left[u, u^{\prime}\right]+\left[v, v^{\prime}\right] \equiv\left[u+v^{\prime}, u^{\prime}+v^{\prime}\right]} \\ c\left[u, u^{\prime}\right] \equiv\left[c u, c u^{\prime}\right] \\ {\left[u, u^{\prime}\right] \cdot\left[v, v^{\prime}\right] \equiv\left[u v, u v^{\prime}+u^{\prime} v\right]} \\ {\left[u, u^{\prime}\right] /\left[v, v^{\prime}\right] \equiv\left[u / v,\left(v u^{\prime}-u v^{\prime}\right) / v^2\right]} \\ \exp (\left[u, u^{\prime}\right]) \equiv\left[e^u, u^{\prime} e^u\right] \\ \ln (\left[u, u^{\prime}\right]) \equiv\left[\ln u_{,} u^{\prime} / u\right] \\ \cos (\left[u, u^{\prime}\right]) \equiv\left[\cos u,-u^{\prime} \sin u^{\prime}\right] \\ \vdots \quad\vdots \end{gathered}$

\begin{aligned} & x \\ & y_1= x^2\\ & y_2=y_1 - x\\ & y_3 = \exp(y_2)\\ & y_4 = y_3/x \end{aligned}

\begin{aligned} & \frac{dy_1}{dx} = 2x\\ &\frac{dy_2}{dx} = \frac{dy_1}{dx} - \frac{dx}{dx} = 2x-1\\ & \frac{dy_3}{dx} = \exp(y_2)\cdot \frac{dy_2}{dx} \\ & \frac{dy_4}{dx} = \frac{\frac{dy_3}{dx}x - y_3}{x^2} \end{aligned}

$(f(g(x)))' = f'(g(x))\cdot g'(x)$

$y_k = g(y_i)$

$y'_k = g'(y_i)\cdot y_i'$

$y_k = g(y_i,\cdots, y_j)$

\begin{aligned} \frac{d}{dx} y_k(x) &= \frac{d}{dx} g(y_i(x),\cdots, y_j(x))\\ &= \sum_{h=i}^j\frac{\partial g}{\partial y_h} \frac{d y_h}{dx} \end{aligned}

\begin{aligned} & y_1 = x\\ & y_2 = x \\ & y_3 = y_2 \\ & y_4 = y_1\cdot y_2\cdot y_3 \\ \end{aligned}

\begin{aligned} \frac{dy_1}{dx} &=1 \\ \frac{dy_2}{dx} &= 1\\ \frac{dy_3}{dx} &= 1\cdot \frac{dy_2}{dx} = 1 \\ \frac{dy_4}{dx} &= y_2 y_3\cdot \frac{dy_1}{dx} + y_1 y_3\frac{dy_2}{dx} + y_1 y_2 \frac{dy_3}{dx}\\ &= y_2 y_3 + y_1 y_3 + y_1y_2 \\ &= 3x^2 \end{aligned}

\begin{aligned} y_1 = \exp(x_2)\\ y_2 = x_1 \cdot y_1\\ y_3 = y_2 - x_1 \end{aligned}

\begin{aligned} & \frac{d y_1}{ d x_2} = \exp(x_2)\\ & \frac{d y_2}{d x_1} = y_1=\exp(x_2) \quad \frac{d y_2}{dx_2} = x_1 \cdot \frac{dy_1}{dx_2} = x_1\cdot \exp(x_2) \\ & \frac{d y_3}{d x_1} = \frac{dy_2}{dx_1} - \frac{dx_1}{dx_1} =\exp(x_2) -1 \quad \frac{dy_3}{dx_2} = \frac{dy_2}{dx_2} = x_1\cdot \exp(x_2) \\ \end{aligned}

import math

class Var:
def __init__(self, val, deriv=1.0):
self.val = val
self.deriv = deriv

if isinstance(other, Var):
val = self.val + other.val
deriv = self.deriv + other.deriv
else:
val = self.val + other
deriv = self.deriv
return Var(val, deriv)

return self + other

def __sub__(self, other):
if isinstance(other, Var):
val = self.val - other.val
deriv = self.deriv - other.deriv
else:
val = self.val - other
deriv = self.deriv
return Var(val, deriv)

def __rsub__(self, other):
val = other - self.val
deriv = - self.deriv
return Var(val, deriv)

def __mul__(self, other):
if isinstance(other, Var):
val = self.val * other.val
deriv = self.val * other.deriv + self.deriv * other.val
else:
val = self.val * other
deriv = self.deriv * other
return Var(val, deriv)

def __rmul__(self, other):
return self * other

def __truediv__(self, other):
if isinstance(other, Var):
val = self.val / other.val
deriv = (self.deriv * other.val - self.val * other.deriv)/other.val**2
else:
val = self.val / other
deriv = self.deriv / other
return Var(val, deriv)

def __rtruediv__(self, other):
val = other / self.val
deriv = other * 1/self.val**2
return Var(val, deriv)

def __repr__(self):
return "value: {}\t gradient: {}".format(self.val, self.deriv)

def exp(f: Var):
return Var(math.exp(f.val), math.exp(f.val) * f.deriv)


fx = lambda x: exp(x*x - x)/x
df = fx(Var(2.0))
print(df)


value: 3.694528049465325         deriv: 9.236320123663312


# 3 反向自动微分

\begin{aligned} y_1 = \exp(x_2)\\ y_2 = x_1 \cdot y_1\\ y_3 = y_2 - x_1 \end{aligned}

\begin{aligned} & \frac{\partial f}{\partial y_3} = 1\\ & \frac{\partial f}{\partial y_2} = \frac{\partial f}{\partial y_3}\frac{\partial y_3}{\partial y_2} = 1 \cdot 1 = 1\\ & \frac{\partial f}{\partial y_1} = \frac{\partial f}{\partial y_2} \frac{\partial y_2}{\partial y_1} = 1 \cdot x_1 = x_1\\ & \frac{\partial f}{\partial x_2} = \frac{\partial f}{\partial y_1} \frac{\partial y_1}{\partial x_2} = x_1 \cdot \exp(x_2)\\ & \frac{\partial f}{\partial x_1} = \frac{\partial f}{\partial y_2}\frac{\partial y_2}{x_1} + \frac{\partial f}{\partial y_3}\frac{\partial y_3}{\partial x_1} = 1\cdot y_1 + 1\cdot (-1) = \exp(x_2) - 1 \end{aligned}

# 参考

• [1] Solomon J. Numerical algorithms: methods for computer vision, machine learning, and graphics[M]. CRC press, 2015.

posted @ 2022-12-28 16:03  orion-orion  阅读(209)  评论(0编辑  收藏  举报