08machine learning cost function for logistic regression

Cost function for logistic regression

how to choose $\vec{w}=[w_1,w_2,w_3...w_n] $ and b?

squared error cost

$J(\vec{w},b)=\frac{1}{m} \sum^{{m}_{i=1}\frac{1}{2}(f_{\vec{w},b}(\vec{x}}-y^{(i)})2 $

we define a loss: $L(f_{\vec{w},b}(\vec{x}^{(i)},y) $

\[L(f_{\vec{w},b}(\vec{x}^{(i)},y^{(i)})= \left\{ \begin{array}{rcl} -log(f_{\vec{w},b}(\vec{x}^{(i)})) \ if \ y^{(i)}=1 \\ -log(1-f_{\vec{\vec{w},b}}(\vec{x}^{(i)}))\ if \ y^{(i)} =0 \end{array}\right. \]

when y = 1,the g(z) is increasingly big,the loss is more little.

when y = 0, the g(z) is smaller, the loss is more little.

the loss is equal to error range.

so we can get a global minimum

our goal was to get a minimum loss .

import numpy as np
%matplotlib widget
import matplotlib.pyplot as plt
from plt_logistic_loss import  plt_logistic_cost, plt_two_logistic_loss_curves, plt_simple_example
from plt_logistic_loss import soup_bowl, plt_logistic_squared_error
plt.style.use('./deeplearning.mplstyle')


x_train = np.array([0., 1, 2, 3, 4, 5],dtype=np.longdouble)
y_train = np.array([0,  0, 0, 1, 1, 1],dtype=np.longdouble)
plt_simple_example(x_train, y_train)

simplified loss function:

\[L(f_{\vec{w},b}(\vec{x}^{(i)}),y^{(i)}) = -y^{(i)}log(f_{\vec{w},b}(\vec{x}^{(i)}))-(1-y^{(i)})log(1-f_{\vec{w},b}(\vec{x}^{(i)})) \\ J(\vec{w},b)=\frac{1}{m}\sum^m_{i=1}[L(f_{\vec{w},b}(\vec{x}^{(i)}),y^{(i)})] \\ = -\frac{1}{m}\sum^m_{i=1}[y^{(i)}log(f_{\vec{w},b}(\vec{x}^{(i)}))+(1-y^{(i)})log(1-f_{\vec{w},b}(\vec{x}^{(i)}))] \]

import numpy as np
%matplotlib widget
import matplotlib.pyplot as plt
from lab_utils_common import  plot_data, sigmoid, dlc
plt.style.use('./deeplearning.mplstyle')

X_train = np.array([[0.5, 1.5], [1,1], [1.5, 0.5], [3, 0.5], [2, 2], [1, 2.5]])  #(m,n)
y_train = np.array([0, 0, 0, 1, 1, 1])   #(m,)


fig,ax = plt.subplots(1,1,figsize=(4,4))
plot_data(X_train, y_train, ax)

# Set both axes to be from 0-4
ax.axis([0, 4, 0, 3.5])
ax.set_ylabel('$x_1$', fontsize=12)
ax.set_xlabel('$x_0$', fontsize=12)
plt.show()

def compute_cost_logistic(X, y, w, b):
    """
    Computes cost

    Args:
      X (ndarray (m,n)): Data, m examples with n features
      y (ndarray (m,)) : target values
      w (ndarray (n,)) : model parameters  
      b (scalar)       : model parameter
      
    Returns:
      cost (scalar): cost
    """

    m = X.shape[0]
    cost = 0.0
    for i in range(m):
        z_i = np.dot(X[i],w) + b
        f_wb_i = sigmoid(z_i)
        cost +=  -y[i]*np.log(f_wb_i) - (1-y[i])*np.log(1-f_wb_i)
             
    cost = cost / m
    return cost

w_tmp = np.array([1,1])
b_tmp = -3
print(compute_cost_logistic(X_train, y_train, w_tmp, b_tmp))



import matplotlib.pyplot as plt

# Choose values between 0 and 6
x0 = np.arange(0,6)

# Plot the two decision boundaries
x1 = 3 - x0
x1_other = 4 - x0

fig,ax = plt.subplots(1, 1, figsize=(4,4))
# Plot the decision boundary
ax.plot(x0,x1, c=dlc["dlblue"], label="$b$=-3")
ax.plot(x0,x1_other, c=dlc["dlmagenta"], label="$b$=-4")
ax.axis([0, 4, 0, 4])

# Plot the original data
plot_data(X_train,y_train,ax)
ax.axis([0, 4, 0, 4])
ax.set_ylabel('$x_1$', fontsize=12)
ax.set_xlabel('$x_0$', fontsize=12)
plt.legend(loc="upper right")
plt.title("Decision Boundary")
plt.show()

w_array1 = np.array([1,1])
b_1 = -3
w_array2 = np.array([1,1])
b_2 = -4

print("Cost for b = -3 : ", compute_cost_logistic(X_train, y_train, w_array1, b_1))
print("Cost for b = -4 : ", compute_cost_logistic(X_train, y_train, w_array2, b_2))

'''
Cost for b = -3 :  0.36686678640551745
Cost for b = -4 :  0.5036808636748461
'''