loss变小梯度是否变小

loss变小梯度反而没有变小。

背景sigmoid(wx+b)的二分类实验，损失函数为平方损失。

实验一:直接训练发现loss减少导致梯度减少。看代码1:

实验二:在wx+b后加上BN层梯loss减少梯度减少。看代码2:

代码1:

from enum import auto

from scipy.io import loadmat

import numpy as np

import torch

import torch.utils.data as data_utils

from torch import nn

import torch.optim as optim

network=nn.Linear(1,1)

tr=torch.Tensor([[-10],[10]])

optimizer = optim.Adam(network.parameters(), lr=0.04)

w=nn.Sigmoid()

i=0

l1=0

while True:

    #i=i+1

    network.train()

    #network1.eval()#

    optimizer.zero_grad()

    #l=w(network(tr))

    l=w(network(tr))

    #print(network1(network(tr)))

    l=(l[0]-0)**2+(l[1]-1)**2

    l.backward()

    optimizer.step()

    print(l)

    if l1==0:

        l1=l

    if l1/l>10:

        for name, parms in network.named_parameters():  

            print('-->name:', name)

            print('-->para:', parms)

            print('-->grad_requirs:',parms.requires_grad)

            print('-->grad_value:',parms.grad)

            print("===")

        l1=l

代码2:

from enum import auto

from scipy.io import loadmat

import numpy as np

import torch

import torch.utils.data as data_utils

from torch import nn

import torch.optim as optim

network=nn.Linear(1,1)

network1=nn.BatchNorm1d(1)

tr=torch.Tensor([[-10],[10]])

optimizer=optim.Adam(network.parameters(), lr=0.04)

optimizer1=optim.Adam(network1.parameters(), lr=0.04)

w=nn.Sigmoid()

i=0

l1=0

while True:

    #i=i+1

    network.train()

    network1.train()#

    optimizer.zero_grad()

    optimizer1.zero_grad()

    #l=w(network(tr))

    l=w(network(tr))

    #print(network1(network(tr)))

    l=(l[0]-0)**2+(l[1]-1)**2

    l.backward()

    optimizer.step()

    optimizer1.step()

    print(l)

    if l1==0:

        l1=l

    if l1/l>10:

        for name, parms in network.named_parameters():  

            print('-->name:', name)

            print('-->para:', parms)

            print('-->grad_requirs:',parms.requires_grad)

            print('-->grad_value:',parms.grad)

            print("===")

        l1=l