import numpy as np
from mlfromscratch.utils import make_diagonal, normalize

# Optimizers for models that use gradient based methods for finding the
# weights that minimizes the loss.
# A great resource for understanding these methods:

def __init__(self, learning_rate=0.01, momentum=0):
self.learning_rate = learning_rate
self.momentum = momentum
self.w_updt = None

# If not initialized
if self.w_updt is None:
self.w_updt = np.zeros(np.shape(w))
# Use momentum if set
self.w_updt = self.momentum * self.w_updt + (1 - self.momentum) * grad_wrt_w
# Move against the gradient to minimize loss
return w - self.learning_rate * self.w_updt

def __init__(self, learning_rate=0.001, momentum=0.4):
self.learning_rate = learning_rate
self.momentum = momentum
self.w_updt = np.array([])

# Calculate the gradient of the loss a bit further down the slope from w
# Initialize on first update
if not self.w_updt.any():
self.w_updt = np.zeros(np.shape(w))

self.w_updt = self.momentum * self.w_updt + self.learning_rate * approx_future_grad
# Move against the gradient to minimize loss
return w - self.w_updt

def __init__(self, learning_rate=0.01):
self.learning_rate = learning_rate
self.G = None # Sum of squares of the gradients
self.eps = 1e-8

# If not initialized
if self.G is None:
self.G = np.zeros(np.shape(w))
# Add the square of the gradient of the loss function at w
return w - self.learning_rate * grad_wrt_w / np.sqrt(self.G + self.eps)

def __init__(self, rho=0.95, eps=1e-6):
self.E_w_updt = None # Running average of squared parameter updates
self.E_grad = None   # Running average of the squared gradient of w
self.w_updt = None   # Parameter update
self.eps = eps
self.rho = rho

# If not initialized
if self.w_updt is None:
self.w_updt = np.zeros(np.shape(w))
self.E_w_updt = np.zeros(np.shape(w))

# Update average of gradients at w

RMS_delta_w = np.sqrt(self.E_w_updt + self.eps)

# Calculate the update

# Update the running average of w updates
self.E_w_updt = self.rho * self.E_w_updt + (1 - self.rho) * np.power(self.w_updt, 2)

return w - self.w_updt

class RMSprop():
def __init__(self, learning_rate=0.01, rho=0.9):
self.learning_rate = learning_rate
self.Eg = None # Running average of the square gradients at w
self.eps = 1e-8
self.rho = rho

# If not initialized
if self.Eg is None:

self.Eg = self.rho * self.Eg + (1 - self.rho) * np.power(grad_wrt_w, 2)

# Divide the learning rate for a weight by a running average of the magnitudes of recent
return w - self.learning_rate *  grad_wrt_w / np.sqrt(self.Eg + self.eps)

def __init__(self, learning_rate=0.001, b1=0.9, b2=0.999):
self.learning_rate = learning_rate
self.eps = 1e-8
self.m = None
self.v = None
# Decay rates
self.b1 = b1
self.b2 = b2

# If not initialized
if self.m is None:

self.m = self.b1 * self.m + (1 - self.b1) * grad_wrt_w
self.v = self.b2 * self.v + (1 - self.b2) * np.power(grad_wrt_w, 2)

m_hat = self.m / (1 - self.b1)
v_hat = self.v / (1 - self.b2)

self.w_updt = self.learning_rate * m_hat / (np.sqrt(v_hat) + self.eps)

return w - self.w_updt

def make_diagonal(x):
""" Converts a vector into an diagonal matrix """
m = np.zeros((len(x), len(x)))
for i in range(len(m[0])):
m[i, i] = x[i]
return m
def normalize(X, axis=-1, order=2):
""" Normalize the dataset X """
l2 = np.atleast_1d(np.linalg.norm(X, order, axis))
l2[l2 == 0] = 1
return X / np.expand_dims(l2, axis)

make_diagonal()的作用是将x中的元素变成对角元素。

normalize()函数的作用是正则化。

• np.linalg.norm(x, ord=None, axis=None, keepdims=False)：需要注意ord的值表示的是范数的类型。
• np.atleast_1d()：改变维度，将输入直接视为1维，比如np.atleast_1d([1])的输出就是[1]
• np.expand_dims()：用于扩展数组的维度，要深入了解还是得去查一下。

class StochasticGradientDescent():
def __init__(self, learning_rate=0.01, momentum=0):
self.learning_rate = learning_rate
self.momentum = momentum
self.w_updt = None

# If not initialized
if self.w_updt is None:
self.w_updt = np.zeros(np.shape(w))
# Use momentum if set
self.w_updt = self.momentum * self.w_updt + (1 - self.momentum) * grad_wrt_w
# Move against the gradient to minimize loss
return w - self.learning_rate * self.w_updt

posted @ 2020-04-16 16:14  西西嘛呦  阅读(1988)  评论(0编辑  收藏  举报