# Source code for pydeep.base.activationfunction

```""" Different kind of non linear activation functions and their derivatives.

:Implemented:

# Unbounded
# Linear
- Identity
# Piecewise-linear
- Rectifier
- RestrictedRectifier (hard bounded)
- LeakyRectifier
# Soft-linear
- ExponentialLinear
- SigmoidWeightedLinear
- SoftPlus
# Bounded
# Step
- Step
# Soft-Step
- Sigmoid
- SoftSign
- HyperbolicTangent
- SoftMax
- K-Winner takes all
# Symmetric, periodic
- Sinus

:Info:
http://en.wikipedia.org/wiki/Activation_function

:Version:
1.1.1

:Date:
16.01.2018

:Author:
Jan Melchior

:Contact:
JanMelchior@gmx.de

This file is part of the Python library PyDeep.

PyDeep is free software: you can redistribute it and/or modify
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program.  If not, see <http://www.gnu.org/licenses/>.

"""
import numpy as numx
from pydeep.base.numpyextension import log_sum_exp

# Unbounded

# Linear

[docs]class Identity(object):
""" Identity function.

:Info: http://www.wolframalpha.com/input/?i=line
"""

[docs]    @classmethod
def f(cls, x):
""" Calculates the identity function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the identity function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return x

[docs]    @classmethod
def g(cls, y):
""" Calculates the inverse identity function value for a given input y.

:param y: Input data.
:type y: scalar or numpy array.

:return: Value of the inverse identity function for y.
:rtype: scalar or numpy array with the same shape as y.
"""
return y

[docs]    @classmethod
def df(cls, x):
""" Calculates the derivative of the identity function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the derivative of the identity function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
if numx.isscalar(x):
return 1.0
else:
return numx.ones(x.shape)

[docs]    @classmethod
def ddf(cls, x):
""" Calculates the second derivative of the identity function value for a given input x.

:param x: Inout data.
:type x: scalar or numpy array.

:return: Value of the second derivative of the identity function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
if numx.isscalar(x):
return 0.0
else:
return numx.zeros(x.shape)

[docs]    @classmethod
def dg(cls, y):
""" Calculates the derivative of the inverse identity function value for a given input y.

:param y: Input data.
:type y: scalar or numpy array.

:return: Value of the derivative of the inverse identity function for y.
:rtype: scalar or numpy array with the same shape as y.
"""
if numx.isscalar(y):
return 1.0
else:
return numx.ones(y.shape)

# Piecewise-linear

[docs]class Rectifier(object):
""" Rectifier activation function function.

:Info: http://www.wolframalpha.com/input/?i=max%280%2Cx%29&dataset=&asynchronous=false&equal=Submit
"""

[docs]    @classmethod
def f(cls, x):
""" Calculates the Rectifier function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the Rectifier function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return numx.maximum(0.0, x)

[docs]    @classmethod
def df(cls, x):
""" Calculates the derivative of the Rectifier function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the derivative of the Rectifier function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return numx.float64(x > 0.0)

[docs]    @classmethod
def ddf(cls, x):
""" Calculates the second derivative of the Rectifier function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the 2nd derivative of the Rectifier function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return 0.0

[docs]class RestrictedRectifier(Rectifier):
""" Restricted Rectifier activation function function.

:Info: http://www.wolframalpha.com/input/?i=max%280%2Cx%29&dataset=&asynchronous=false&equal=Submit
"""

[docs]    def __init__(self, restriction=1.0):
""" Constructor.

:param restriction: Restriction value / upper limit value.
:type restriction: float.
"""
self.restriction = restriction

[docs]    def f(self, x):
""" Calculates the Restricted Rectifier function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the Restricted Rectifier function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return numx.minimum(numx.maximum(0.0, x), self.restriction)

[docs]    def df(self, x):
""" Calculates the derivative of the Restricted Rectifier function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array

:return: Value of the derivative of the Restricted Rectifier function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return numx.float64(x > 0.0) * numx.float64(x < self.restriction)

[docs]class LeakyRectifier(Rectifier):
""" Leaky Rectifier activation function function.

:Info: https://en.wikipedia.org/wiki/Activation_function
"""

[docs]    def __init__(self, negativeSlope=0.01, positiveSlope=1.0):
""" Constructor.

:param negativeSlope: Slope when x < 0
:type negativeSlope: scalar

:param positiveSlope: Slope when x >= 0
:type positiveSlope: scalar

"""
self.negativeSlope = negativeSlope
self.positiveSlope = positiveSlope

[docs]    def f(self, x):
""" Calculates the Leaky Rectifier function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the Leaky Rectifier function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return x*((x < 0)*(self.negativeSlope-self.positiveSlope) + self.positiveSlope)

[docs]    def df(self, x):
""" Calculates the derivative of the Leaky Rectifier function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array

:return: Value of the derivative of the Leaky Rectifier function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return (x < 0)*(self.negativeSlope-self.positiveSlope) + self.positiveSlope

# Soft-linear

[docs]class ExponentialLinear(object):
""" Exponential Linear activation function function.

:Info: https://en.wikipedia.org/wiki/Activation_function
"""

[docs]    def __init__(self, alpha=1.0):
""" Constructor.

:param alpha: scaling factor
:type alpha: scalar

"""
self.alpha = alpha

[docs]    def f(self, x):
""" Calculates the Exponential Linear function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the Exponential Linear function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
decision = x > 0.0
return x*decision+(1.0-decision)*self.alpha*(numx.exp(x)-1.0)

[docs]    def df(self, x):
""" Calculates the derivative of the Exponential Linear function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the derivative of the Exponential Linear function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
decision = x > 0.0
return decision+(1.0-decision)*self.alpha*numx.exp(x)

[docs]class SigmoidWeightedLinear(object):
""" Sigmoid weighted linear units (also named Swish)

:Info: https://arxiv.org/pdf/1702.03118v1.pdf and for Swish: https://arxiv.org/pdf/1710.05941.pdf
"""

[docs]    def __init__(self, beta=1.0):
""" Constructor.

:param beta: scaling factor
:type beta: scalar

"""
self.beta = beta

[docs]    def f(self, x):
""" Calculates the Sigmoid weighted linear function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the Sigmoid weighted linear function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return x*Sigmoid.f(self.beta*x)

[docs]    def df(self, x):
""" Calculates the derivative of the Sigmoid weighted linear function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the derivative of the Sigmoid weighted linear function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
sig = Sigmoid.f(self.beta*x)
return sig*(1.0+x*(1.0-sig))

[docs]class SoftPlus(object):
""" Soft Plus function.

:Info: http://www.wolframalpha.com/input/?i=log%28exp%28x%29%2B1%29
"""

[docs]    @classmethod
def f(cls, x):
""" Calculates the SoftPlus function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the SoftPlus function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return numx.log(1.0 + numx.exp(x))

[docs]    @classmethod
def g(cls, y):
""" Calculates the inverse SoftPlus function value for a given input y.

:param y: Input data.
:type y: scalar or numpy array.

:return: Value of the inverse SoftPlus function for y.
:rtype: scalar or numpy array with the same shape as y.
"""
return numx.log(numx.exp(y) - 1.0)

[docs]    @classmethod
def df(cls, x):
""" Calculates the derivative of the SoftPlus function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the derivative of the SoftPlus function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return 1.0 / (1.0 + numx.exp(-x))

[docs]    @classmethod
def ddf(cls, x):
""" Calculates the second derivative of the SoftPlus function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array

:return: Value of the 2nd derivative of the SoftPlus function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
exp_x = numx.exp(x)
return exp_x / ((1.0 + exp_x) ** 2)

[docs]    @classmethod
def dg(cls, y):
""" Calculates the derivative of the inverse SoftPlus function value for a given input y.

:param y: Input data.
:type y: scalar or numpy array.

:return: Value of the derivative of the inverse SoftPlus function for x.
:rtype: scalar or numpy array with the same shape as y.
"""
return 1.0 / (1.0 - numx.exp(-y))

# Bounded

# Step

[docs]class Step(object):
""" Step activation function function.
"""

[docs]    @classmethod
def f(cls, x):
""" Calculates the step function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the step function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return numx.float64(x > 0)

[docs]    @classmethod
def df(cls, x):
""" Calculates the derivative of the step function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the derivative of the step function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return 0.0

[docs]    @classmethod
def ddf(cls, x):
""" Calculates the second derivative of the step function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the derivative of the Step function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return 0.0

# Soft-step

[docs]class Sigmoid(object):
""" Sigmoid function.

:Info: http://www.wolframalpha.com/input/?i=sigmoid
"""

[docs]    @classmethod
def f(cls, x):
""" Calculates the Sigmoid function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the Sigmoid function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return 0.5 + 0.5 * numx.tanh(0.5 * x)

[docs]    @classmethod
def g(cls, y):
""" Calculates the inverse Sigmoid function value for a given input y.

:param y: Input data.
:type y: scalar or numpy array.

:return: Value of the inverse Sigmoid function for y.
:rtype: scalar or numpy array with the same shape as y.
"""
return 2.0 * numx.arctanh(2.0 * y - 1.0)

[docs]    @classmethod
def df(cls, x):
""" Calculates the derivative of the Sigmoid function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the derivative of the Sigmoid function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
# expx = numx.exp(-x)
# return numx.exp(-x)/((1+numx.exp(-x))**2)
sig = cls.f(x)
return sig * (1.0 - sig)

[docs]    @classmethod
def ddf(cls, x):
""" Calculates the second derivative of the Sigmoid function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the second derivative of the Sigmoid function for x.
:rtype:  scalar or numpy array with the same shape as x.
"""
sig = cls.f(x)
return sig - 3 * (sig ** 2) + 2 * (sig ** 3)

[docs]    @classmethod
def dg(cls, y):
""" Calculates the derivative of the inverse Sigmoid function value for a given input y.

:param y: Input data.
:type y: scalar or numpy array.

:return: Value of the derivative of the inverse Sigmoid function for y.
:rtype: scalar or numpy array with the same shape as y.
"""
return 1.0 / (y - y ** 2)

[docs]class SoftSign(object):
""" SoftSign function.

:Info: http://www.wolframalpha.com/input/?i=x%2F%281%2Babs%28x%29%29
"""

[docs]    @classmethod
def f(cls, x):
""" Calculates the SoftSign function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the SoftSign function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return x / (1.0 + numx.abs(x))

[docs]    @classmethod
def df(cls, x):
""" Calculates the derivative of the SoftSign function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the SoftSign function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return 1.0 / ((1.0 + numx.abs(x)) ** 2)

[docs]    @classmethod
def ddf(cls, x):
""" Calculates the second derivative of the SoftSign function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the 2nd derivative of the SoftSign function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
absx = numx.abs(x)
return -(2.0 * x) / (absx * (1 + absx) ** 3)

[docs]class HyperbolicTangent(object):
""" HyperbolicTangent function.

:Info: http://www.wolframalpha.com/input/?i=tanh
"""

[docs]    @classmethod
def f(cls, x):
""" Calculates the Hyperbolic Tangent function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the Hyperbolic Tangent function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return numx.tanh(x)

[docs]    @classmethod
def g(cls, y):
""" Calculates the inverse Hyperbolic Tangent function value for a given input y.

:param y: Input data.
:type y: scalar or numpy array.

:return: alue of the inverse Hyperbolic Tangent function for y.
:rtype: scalar or numpy array with the same shape as x.
"""
return 0.5 * (numx.log(1.0 + y) - numx.log(1.0 - y))

[docs]    @classmethod
def df(cls, x):
""" Calculates the derivative of the Hyperbolic Tangent function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the derivative of the Hyperbolic Tangent function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
tanh = cls.f(x)
return 1.0 - tanh ** 2

[docs]    @classmethod
def ddf(cls, x):
""" Calculates the second derivative of the Hyperbolic Tangent function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array.

:return: Value of the second derivative of the Hyperbolic Tangent function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
tanh = cls.f(x)
return -2 * tanh * (1 - (tanh ** 2))

[docs]    @classmethod
def dg(cls, y):
""" Calculates the derivative of the inverse Hyperbolic Tangent function value for a given input y.

:param y: Input data.
:type y: scalar or numpy array.

:return: Value the derivative of the inverse Hyperbolic Tangent function for x.
:rtype: scalar or numpy array with the same shape as y.
"""
return numx.exp(-numx.log((1.0 - y ** 2)))

[docs]class SoftMax(object):
""" Soft Max function.

:Info: https://en.wikipedia.org/wiki/Activation_function
"""

[docs]    @classmethod
def f(cls, x):
""" Calculates the function value of the SoftMax function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array

:return: Value of the SoftMax function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return numx.exp(x - log_sum_exp(x, axis=1).reshape(x.shape[0], 1))

[docs]    @classmethod
def df(cls, x):
""" Calculates the derivative of the SoftMax function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array

:return: Value of the derivative of the SoftMax function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
result = x[0] * numx.eye(x.shape[1], x.shape[1]) - numx.dot(x[0].reshape(x.shape[1], 1), x[0]
.reshape(1, x.shape[1])
).reshape(1, x.shape[1], x.shape[1])
for i in range(1, x.shape[0], 1):
result = numx.vstack(
(result, x[i] * numx.eye(x.shape[1], x.shape[1]) - numx.dot(x[i].reshape(x.shape[1], 1),
x[i].reshape(1, x.shape[1])
).reshape(1, x.shape[1], x.shape[1])))
''' WITHOUT LOOP BUT MUCH MOR MEMORY CONSUMPTION
result = x.reshape((1, 100*100))
result = numx.tile(result, (100, 1))
result_t = result.T
result_t = numx.array_split(result_t, 100)
result_t = numx.hstack(result_t)
result *= (numx.tile( numx.eye(100), (1, 100)) - result_t)
result *= numx.tile(y.reshape((1, 100*100)), (100, 1))
result = numx.sum(result, axis=0)
'''
return result

# Symmetric, periodic

:Info: http://www.wolframalpha.com/input/?i=Gaussian
"""

[docs]    def __init__(self, mean=0.0, variance=1.0):
""" Constructor.

:param mean: Mean of the function.
:type mean: scalar or numpy array

:param variance: Variance of the function.
:type variance: scalar or numpy array
"""
self.mean = mean
self.variance = variance

[docs]    def f(self, x):
""" Calculates the Radial Basis function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array

:return: Value of the Radial Basis function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
activation = x - self.mean
return numx.exp(-(activation ** 2) / self.variance)

[docs]    def df(self, x):
""" Calculates the derivative of the Radial Basis function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array

:return: Value of the derivative of the Radial Basis function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return (self.f(x) * 2 * (self.mean - x)) / self.variance

[docs]    def ddf(self, x):
""" Calculates the second derivative of the Radial Basis function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array

:return: Value of the second derivative of the Radial Basis function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
activation = ((x - self.mean) ** 2) / self.variance
return 2.0 / self.variance * numx.exp(-activation) * (2 * activation - 1.0)

[docs]class Sinus(object):
""" Sinus function.

:Info: http://www.wolframalpha.com/input/?i=sin(x)
"""

[docs]    @classmethod
def f(cls, x):
""" Calculates the function value of the Sinus function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array

:return: Value of the Sinus function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return numx.sin(x)

[docs]    @classmethod
def df(cls, x):
""" Calculates the derivative of the Sinus function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array

:return: Value of the derivative of the Sinus function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return numx.cos(x)

[docs]    @classmethod
def ddf(cls, x):
""" Calculates the second derivative of the Sinus function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array

:return: Value of the second derivative of the Sinus function for x.
:rtype: scalar or numpy array with the same shape as x.
"""
return -numx.sin(x)

[docs]class KWinnerTakeAll(object):
""" K Winner take all activation function.

:WARNING: The derivative gets already calcluated in the forward pass.
Thus, for the same data-point the order should always be forward_pass, backward_pass!

"""

[docs]    def __init__(self, k, axis=1, activation_function=Identity()):
""" Constructor.

:param k: Number of active units.
:type k: int

:param axis: Axis to compute the maximum.
:type axis: int

:param k: activation_function
:type k: Instance of an activation function

"""
self.k = k
self.axis = axis
self.activation_function = activation_function
self._temp_derivative = None

[docs]    def f(self, x):
""" Calculates the K-max function value for a given input x.

:param x: Input data.
:type x: scalar or numpy array

:return: Value of the Kmax function for x.
:rtype: scalar or numpy array with the same shape as x.

"""
act = self.activation_function.f(numx.atleast_2d(x))
winner = None
if self.axis == 0:
winner = numx.float64(act >= numx.atleast_2d(numx.sort(act, axis=self.axis)[-self.k, :]))
else:
winner = numx.float64(act.T >= numx.atleast_2d(numx.sort(act, axis=self.axis)[:, -self.k])).T
self._temp_derivative = winner * self.activation_function.df(x)
return act * winner

[docs]    def df(self, x):
""" Calculates the derivative of the KWTA function.

:param x: Input data.
:type x: scalar or numpy array

:return: Derivative of the KWTA function
:rtype: scalar or numpy array with the same shape as x.

"""
return self._temp_derivative
```