# 改进版梯度下降

## 改进版梯度下降

from matplotlib import pyplot as plt
import numpy as np
from mpl_toolkits.mplot3d import Axes3D

fig = plt.figure()
x = np.arange(-4, 4, 0.025)
plt.plot(x,x**2)
plt.title("y = x^2")
def f(x):
return x**2
def h(x):
return 2*x
η = 0.25
ε = 0.1
x = 4
iters = 0
X = []
Y = []
while iters<12:
iters+=1
X.append(x)
Y.append(f(x))
print(iters,x)
plt.plot(X,Y,"ro")
ax = plt.subplot()
for i in range(len(X)):
ax.text(X[i], (X[i])**2, "({:.3f},{:.3f})".format(X[i], (X[i])**2), color='red')
plt.show()

### RMSprop法

from matplotlib import pyplot as plt
import numpy as np
from mpl_toolkits.mplot3d import Axes3D

fig = plt.figure()
x = np.arange(-4, 4, 0.025)
plt.plot(x,x**2)
plt.title("y = x^2")
def f(x):
return x**2
def h(x):
return 2*x
g = 1
x = 4
ρ = 0.9
η = 0.01
ε = 10e-10
iters = 0
X = []
Y = []
while iters<12:
iters+=1
X.append(x)
Y.append(f(x))
g = ρ*g+(1-ρ)*h(x)**2
x = x - η/np.sqrt(g+ε)*h(x)
print(iters,x)
plt.plot(X,Y,"ro")
ax = plt.subplot()
for i in range(len(X)):
ax.text(X[i], (X[i])**2, "({:.3f},{:.3f})".format(X[i], (X[i])**2), color='red')
plt.show()

### Momentum法

Momentum是动量的意思，想象一下，一个小车从高坡上冲下来，他不会停在最低点，因为他还有一个动量，还会向前冲，甚至可以冲过一些小的山丘，如果面对的是较大的坡，他可能爬不上去，最终又会倒车回来，折叠几次，停在谷底。

v表示动量，初始v=0

α是一个接近于1的数，一般设置为0.9，也就是把之前的动量缩减到0.9倍

η是学习率

from matplotlib import pyplot as plt
import numpy as np

fig = plt.figure()
x = np.arange(-0.8, 1.2, 0.025)
plt.plot(x,-x**3-x**2+2*x**4)
plt.title("y = 2*x^4-x^3-x^2")
def f(x):
return 2*x**4-x**3-x**2
def h(x):
return 8*x**3 - 3*x**2 - 2*x
η = 0.05
α = 0.9
v = 0
x = -0.8
iters = 0
X = []
Y = []
while iters<12:
iters+=1
X.append(x)
Y.append(f(x))
v = α*v - η*h(x)
x = x + v
print(iters,x)
plt.plot(X,Y)
plt.show()

③和④式目的是纠正偏差

⑤式进行梯度更新

from matplotlib import pyplot as plt
import numpy as np
from mpl_toolkits.mplot3d import Axes3D

fig = plt.figure()
x = np.arange(-4, 4, 0.025)
plt.plot(x,x**2)
plt.title("y = x^2")
def f(x):
return x**2
def h(x):
return 2*x
x = 4
m = 0
v = 0
β1 = 0.9
β2 = 0.999
η = 0.001
ε = 10e-8
iters = 0
X = []
Y = []
while iters<12:
iters+=1
X.append(x)
Y.append(f(x))
m = β1*m + (1-β1)*h(x)
v = β2*v + (1-β2)*h(x)**2
m_het = m/(1-β1**iters)
v_het = v/(1-β2**iters)
x = x - η/np.sqrt(v_het+ε)*m_het
print(iters,x)
plt.plot(X,Y,"ro")
ax = plt.subplot()
for i in range(len(X)):
ax.text(X[i], (X[i])**2, "({:.3f},{:.3f})".format(X[i], (X[i])**2), color='red')
plt.show()

posted @ 2019-09-10 08:28  Timcode  阅读(...)  评论(...编辑  收藏