# 【python实现卷积神经网络】激活函数的实现（sigmoid、softmax、tanh、relu、leakyrelu、elu、selu、softplus）

import numpy as np

# Collection of activation functions
# Reference: https://en.wikipedia.org/wiki/Activation_function

class Sigmoid():
def __call__(self, x):
return 1 / (1 + np.exp(-x))

return self.__call__(x) * (1 - self.__call__(x))

class Softmax():
def __call__(self, x):
e_x = np.exp(x - np.max(x, axis=-1, keepdims=True))
return e_x / np.sum(e_x, axis=-1, keepdims=True)

p = self.__call__(x)
return p * (1 - p)

class TanH():
def __call__(self, x):
return 2 / (1 + np.exp(-2*x)) - 1

return 1 - np.power(self.__call__(x), 2)

class ReLU():
def __call__(self, x):
return np.where(x >= 0, x, 0)

return np.where(x >= 0, 1, 0)

class LeakyReLU():
def __init__(self, alpha=0.2):
self.alpha = alpha

def __call__(self, x):
return np.where(x >= 0, x, self.alpha * x)

return np.where(x >= 0, 1, self.alpha)

class ELU():
def __init__(self, alpha=0.1):
self.alpha = alpha

def __call__(self, x):
return np.where(x >= 0.0, x, self.alpha * (np.exp(x) - 1))

return np.where(x >= 0.0, 1, self.__call__(x) + self.alpha)

class SELU():
# Reference : https://arxiv.org/abs/1706.02515,
# https://github.com/bioinf-jku/SNNs/blob/master/SelfNormalizingNetworks_MLP_MNIST.ipynb
def __init__(self):
self.alpha = 1.6732632423543772848170429916717
self.scale = 1.0507009873554804934193349852946

def __call__(self, x):
return self.scale * np.where(x >= 0.0, x, self.alpha*(np.exp(x)-1))

return 1 / (1 + np.exp(-x))