pytorch-day04

1、梯度

导数：因变量随着自变量的变化而变化的趋势（即函数的变化量）。导数的范围比较广，可以说x的方向，也可以说y的方向。

　偏微分：只指变量的方向。偏微分和导数一样，只不过偏微分指定了方向（变量的方向，即有多少个变量就有多少个偏微分）

　梯度：所有偏微分的一个向量（dx，dy，......，dz）

　凸函数能找到一个全局最优解。

　Resnet-56

　鞍点

　初始值、学习率、动量

2、常见函数梯度

3、激活函数梯度

4、LOSS及其梯度

　使用pytorch对LOSS函数进行求导：

import torch
from torch.nn import functional as F

# method1:
x = torch.ones(1)
# w = torch.full([1], 2)
# w.requires_grad_()
w = torch.tensor(torch.full([1], 2), requires_grad=True)
mse = F.mse_loss(torch.ones(1), x * w)
# 损失函数对w求梯度
dw = torch.autograd.grad(mse, [w])
print(dw)  # (tensor([2.]),)

# method2:
x = torch.ones(1)
w = torch.tensor(torch.full([1], 2), requires_grad=True)
mse = F.mse_loss(torch.ones(1), x * w)
# 反向传播的时候，会计算每个Tensor的grad(grad不会当做返回值返回,而是会附加在每个Tensor的成员变量上,需要使用.获取)
mse.backward()
print(w.grad)  # tensor([2.])
# 当grad太大的时候，我们一般不直接看grad,而是看它的范数(norm)

# Softmax激活函数
a = torch.tensor(torch.rand(3), requires_grad=True)
p = F.softmax(a, dim=0)
c = torch.autograd.grad(p[0], [a], retain_graph=True)
d = torch.autograd.grad(p[1], [a], retain_graph=True)
e = torch.autograd.grad(p[2], [a], retain_graph=True)
print(c)  # (tensor([ 0.2427, -0.1260, -0.1167]),)
print(d)  # (tensor([-0.1260,  0.2116, -0.0856]),)
print(e)  # (tensor([-0.1167, -0.0856,  0.2022]),)

5、感知机的梯度推导

　感知机是单输出

　单输出：

# 损失函数对w求梯度，单输出
x = torch.randn(1, 10)
w = torch.randn(1, 10, requires_grad=True)
o = torch.sigmoid(x @ w.t())

loss = F.mse_loss(torch.ones(1, 1), o)
loss.backward()
print(w.grad)  # dw0 dw1 ... dw9

　多输出：

# 损失函数对w求梯度,多输出
x = torch.randn(1, 10)
w = torch.randn(2, 10, requires_grad=True)
o = torch.sigmoid(x @ w.t())
print(o)
loss = F.mse_loss(torch.ones(1, 1), o)  # 用来Broadcasting机制
# loss = F.mse_loss(torch.ones(1, 2), o) # 写2更适合
print(loss)
loss.backward()
print(w.grad)

6、链式法则

# 链式法则
x = torch.tensor(1.)
w1 = torch.tensor(2., requires_grad=True)
b1 = torch.tensor(1.)
w2 = torch.tensor(2., requires_grad=True)
b2 = torch.tensor(1.)

y1 = x * w1 + b1
y2 = y1 * w2 + b2
# 使用链式法则求导
dy2_dy1 = autograd.grad(y2, [y1], retain_graph=True)[0]
dy1_dw1 = autograd.grad(y1, [w1], retain_graph=True)[0]
# 使用pytorch求导
dy2_dw1 = autograd.grad(y2, [w1], retain_graph=True)[0]
# 结果相同
print(dy2_dw1 * dy1_dw1)  # tensor(2.)
print(dy2_dw1)  # tensor(2.)

7、MLP（多层感知机）反向传播

import numpy as np
from mpl_toolkits.mplot3d import Axes3D
from matplotlib import pyplot as plt
import torch

def himmelblau(x):
    return (x[0] ** 2 + x[1] - 11) ** 2 + (x[0] + x[1] ** 2 - 7) ** 2

x = np.arange(-6, 6, 0.1)
y = np.arange(-6, 6, 0.1)
print('x,y range:', x.shape, y.shape)
X, Y = np.meshgrid(x, y)
print('X,Y maps:', X.shape, Y.shape)
Z = himmelblau([X, Y])

fig = plt.figure('himmelblau')
ax = fig.gca(projection='3d')
ax.plot_surface(X, Y, Z)
ax.view_init(60, -30)
ax.set_xlabel('x')
ax.set_ylabel('y')
plt.show()

# [1., 0.], [-4, 0.], [4, 0.]
x = torch.tensor([1., 0.], requires_grad=True)  # 起始点
optimizer = torch.optim.Adam([x], lr=1e-3)
# 优化的目标是x向量,这句code的意思是定义优化的目标
# 执行optimizer.step()时,相当于x[0]'=x[0]-lr*(dz/dx[0])和x[1]'=x[1]-lr*(dz/dx[1])

for step in range(20000):

    pred = himmelblau(x)

    optimizer.zero_grad()
    pred.backward()  # 求梯度
    optimizer.step()  # 迭代更新

    if step % 2000 == 0:
        print('step {}: x = {}, f(x) = {}'
              .format(step, x.tolist(), pred.item()))

8、Logistic Regression

9、交叉熵（不确定性）

　与softmax匹配的loss：cross entropy loss

# 交叉熵
x = torch.randn(1, 784)
w = torch.randn(10, 784)
logits = x @ w.t()
pre = F.softmax(logits, dim=1)  # 自己实现
pred_log = torch.log(pre)  # 自己实现
f1 = F.nll_loss(pred_log, torch.tensor([3]))  # 自己实现

f2 = F.cross_entropy(logits, torch.tensor([3]))  # 不需要自己实现，之间调用
# 也就是说cross_entropy=softmax+log+nll_loss
print(f1)  # tensor(0.1039)
print(f2)  # tensor(0.1039)

10、LR多分类实战（底层实现）

 1 import torch
 2 import torch.nn as nn
 3 import torch.nn.functional as F
 4 import torch.optim as optim
 5 from torchvision import datasets, transforms
 6 
 7 batch_size = 200
 8 learning_rate = 0.01
 9 epochs = 10
10 
11 train_loader = torch.utils.data.DataLoader(
12     datasets.MNIST('../data', train=True, download=True,
13                    transform=transforms.Compose([
14                        transforms.ToTensor(),
15                        transforms.Normalize((0.1307,), (0.3081,))
16                    ])),
17     batch_size=batch_size, shuffle=True)
18 test_loader = torch.utils.data.DataLoader(
19     datasets.MNIST('../data', train=False, transform=transforms.Compose([
20         transforms.ToTensor(),
21         transforms.Normalize((0.1307,), (0.3081,))
22     ])),
23     batch_size=batch_size, shuffle=True)
24  # 1、手动定义参数
25 w1, b1 = torch.randn(200, 784, requires_grad=True), \
26          torch.zeros(200, requires_grad=True)
27 w2, b2 = torch.randn(200, 200, requires_grad=True), \
28          torch.zeros(200, requires_grad=True)
29 w3, b3 = torch.randn(10, 200, requires_grad=True), \
30          torch.zeros(10, requires_grad=True)
31 #初始化
32 torch.nn.init.kaiming_normal_(w1)
33 torch.nn.init.kaiming_normal_(w2)
34 torch.nn.init.kaiming_normal_(w3)
35 
36 
37 def forward(x): # 2、手动编写网络层
38     x = x @ w1.t() + b1
39     x = F.relu(x)
40     x = x @ w2.t() + b2
41     x = F.relu(x)
42     x = x @ w3.t() + b3
43     x = F.relu(x)
44     return x
45 
46 
47 optimizer = optim.SGD([w1, b1, w2, b2, w3, b3], lr=learning_rate)
48 criteon = nn.CrossEntropyLoss()  #定义损失函数
49 
50 for epoch in range(epochs):
51 
52     for batch_idx, (data, target) in enumerate(train_loader):
53         data = data.view(-1, 28 * 28)
54 
55         logits = forward(data)
56         loss = criteon(logits, target)
57 
58         optimizer.zero_grad()
59         loss.backward()
60         # print(w1.grad.norm(), w2.grad.norm())
61         optimizer.step()
62 
63         if batch_idx % 100 == 0:
64             print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
65                 epoch, batch_idx * len(data), len(train_loader.dataset),
66                        100. * batch_idx / len(train_loader), loss.item()))
67 
68     test_loss = 0
69     correct = 0
70     for data, target in test_loader:
71         data = data.view(-1, 28 * 28)
72         logits = forward(data)
73         test_loss += criteon(logits, target).item()
74 
75         pred = logits.data.max(1)[1]
76         correct += pred.eq(target.data).sum()
77 
78     test_loss /= len(test_loader.dataset)
79     print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
80         test_loss, correct, len(test_loader.dataset),
81         100. * correct / len(test_loader.dataset)))

11、全连接层（使用API实现）

 1 import torch
 2 import torch.nn as nn
 3 import torch.nn.functional as F
 4 import torch.optim as optim
 5 from torchvision import datasets, transforms
 6 
 7 batch_size = 200
 8 learning_rate = 0.01
 9 epochs = 10
10 
11 train_loader = torch.utils.data.DataLoader(
12     datasets.MNIST('../data', train=True, download=True,
13                    transform=transforms.Compose([
14                        transforms.ToTensor(),
15                        transforms.Normalize((0.1307,), (0.3081,))
16                    ])),
17     batch_size=batch_size, shuffle=True)
18 test_loader = torch.utils.data.DataLoader(
19     datasets.MNIST('../data', train=False, transform=transforms.Compose([
20         transforms.ToTensor(),
21         transforms.Normalize((0.1307,), (0.3081,))
22     ])),
23     batch_size=batch_size, shuffle=True)
24 
25 
26 class MLP(nn.Module):
27     def __init__(self):
28         super(MLP, self).__init__()
29 
30         self.model = nn.Sequential(
31             nn.Linear(784, 200),
32             nn.ReLU(inplace=True),
33             nn.Linear(200, 200),
34             nn.ReLU(inplace=True),
35             nn.Linear(200, 10),
36             nn.ReLU(inplace=True),
37         )
38 
39     def forward(self, x):
40         x = self.model(x)
41         return x
42 
43 
44 net = MLP()
45 optimizer = optim.SGD(net.parameters(), lr=learning_rate) # 不需要自己初始化参数
46 criteon = nn.CrossEntropyLoss()
47 
48 for epoch in range(epochs):
49 
50     for batch_idx, (data, target) in enumerate(train_loader):
51         data = data.view(-1, 28 * 28)
52 
53         logits = net(data)  # 实际上module(data)  等价于module.forward(data)
54         loss = criteon(logits, target)
55 
56         optimizer.zero_grad()
57         loss.backward()
58         # print(w1.grad.norm(), w2.grad.norm())
59         optimizer.step()
60 
61         if batch_idx % 100 == 0:
62             print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
63                 epoch, batch_idx * len(data), len(train_loader.dataset),
64                        100. * batch_idx / len(train_loader), loss.item()))
65 
66     test_loss = 0
67     correct = 0
68     for data, target in test_loader:
69         data = data.view(-1, 28 * 28)
70         logits = net(data)
71         test_loss += criteon(logits, target).item()
72 
73         pred = logits.data.max(1)[1]
74         correct += pred.eq(target.data).sum()
75 
76     test_loss /= len(test_loader.dataset)
77     print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
78         test_loss, correct, len(test_loader.dataset),
79         100. * correct / len(test_loader.dataset)))

12、激活函数与GPU加速

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
from torchvision import datasets, transforms

batch_size = 200
learning_rate = 0.01
epochs = 10

train_loader = torch.utils.data.DataLoader(
    datasets.MNIST('../data', train=True, download=True,
                   transform=transforms.Compose([
                       transforms.ToTensor(),
                       transforms.Normalize((0.1307,), (0.3081,))
                   ])),
    batch_size=batch_size, shuffle=True)
test_loader = torch.utils.data.DataLoader(
    datasets.MNIST('../data', train=False, transform=transforms.Compose([
        transforms.ToTensor(),
        transforms.Normalize((0.1307,), (0.3081,))
    ])),
    batch_size=batch_size, shuffle=True)


class MLP(nn.Module):
    def __init__(self):
        super(MLP, self).__init__()

        self.model = nn.Sequential(
            nn.Linear(784, 200),
            nn.LeakyReLU(inplace=True),
            nn.Linear(200, 200),
            nn.LeakyReLU(inplace=True),
            nn.Linear(200, 10),
            nn.LeakyReLU(inplace=True),
        )

    def forward(self, x):
        x = self.model(x)
        return x


"""
    注意：model.to(device)和data.to(device)不同
    model.to(device):是把模型搬到cuda上，始终只有一个
    data.to(device):从新创建一个cuda tensor,而不是将之前的cpu上的tensor搬到cuda上
"""
device = torch.device('cuda:0')
net = MLP().to(device)
optimizer = optim.SGD(net.parameters(), lr=learning_rate)
criteon = nn.CrossEntropyLoss().to(device)

for epoch in range(epochs):

    for batch_idx, (data, target) in enumerate(train_loader):
        data = data.view(-1, 28 * 28)
        data, target = data.to(device), target.cuda()  # cuda()已经不推荐使用了，推荐使用to(device)

        logits = net(data)
        loss = criteon(logits, target)

        optimizer.zero_grad()
        loss.backward()
        # print(w1.grad.norm(), w2.grad.norm())
        optimizer.step()

        if batch_idx % 100 == 0:
            print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
                epoch, batch_idx * len(data), len(train_loader.dataset),
                       100. * batch_idx / len(train_loader), loss.item()))

    test_loss = 0
    correct = 0
    for data, target in test_loader:
        data = data.view(-1, 28 * 28)
        data, target = data.to(device), target.cuda()
        logits = net(data)
        test_loss += criteon(logits, target).item()

        pred = logits.data.max(1)[1]
        correct += pred.eq(target.data).sum()

    test_loss /= len(test_loader.dataset)
    print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
        test_loss, correct, len(test_loader.dataset),
        100. * correct / len(test_loader.dataset)))

13、MNIST测试

 1 import torch
 2 import torch.nn as nn
 3 import torch.nn.functional as F
 4 import torch.optim as optim
 5 from torchvision import datasets, transforms
 6 
 7 batch_size = 200
 8 learning_rate = 0.01
 9 epochs = 10
10 
11 train_loader = torch.utils.data.DataLoader(
12     datasets.MNIST('../data', train=True, download=True,
13                    transform=transforms.Compose([
14                        transforms.ToTensor(),
15                        transforms.Normalize((0.1307,), (0.3081,))
16                    ])),
17     batch_size=batch_size, shuffle=True)
18 test_loader = torch.utils.data.DataLoader(
19     datasets.MNIST('../data', train=False, transform=transforms.Compose([
20         transforms.ToTensor(),
21         transforms.Normalize((0.1307,), (0.3081,))
22     ])),
23     batch_size=batch_size, shuffle=True)
24 
25 
26 class MLP(nn.Module):
27     def __init__(self):
28         super(MLP, self).__init__()
29 
30         self.model = nn.Sequential(
31             nn.Linear(784, 200),
32             nn.LeakyReLU(inplace=True),
33             nn.Linear(200, 200),
34             nn.LeakyReLU(inplace=True),
35             nn.Linear(200, 10),
36             nn.LeakyReLU(inplace=True),
37         )
38 
39     def forward(self, x):
40         x = self.model(x)
41         return x
42 
43 
44 net = MLP()
45 optimizer = optim.SGD(net.parameters(), lr=learning_rate)
46 criteon = nn.CrossEntropyLoss()
47 
48 for epoch in range(epochs):
49     for batch_idx, (data, target) in enumerate(train_loader):
50         data = data.view(-1, 28 * 28)
51 
52         logits = net(data)
53         loss = criteon(logits, target)
54 
55         optimizer.zero_grad()
56         loss.backward()
57         # print(w1.grad.norm(), w2.grad.norm())
58         optimizer.step()
59 
60         if batch_idx % 100 == 0:
61             print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
62                 epoch, batch_idx * len(data), len(train_loader.dataset),
63                        100. * batch_idx / len(train_loader), loss.item()))
64 
65     # 一个epoch做一个test
66     test_loss = 0
67     correct = 0
68     for data, target in test_loader:
69         data = data.view(-1, 28 * 28)
70         logits = net(data)
71         test_loss += criteon(logits, target).item()
72 
73         pred = logits.argmax(dim=1)  # 返回索引号
74         correct += pred.eq(target).float().sum().item()  # 正确预测的个数
75 
76     test_loss /= len(test_loader.dataset)
77     print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
78         test_loss, correct, len(test_loader.dataset),
79         100. * correct / len(test_loader.dataset)))

14、Visdom可视化（pip install visdom）

　访问图片的网页：

　执行下面的代码，然后在观察网页：

 1 import torch
 2 import torch.nn as nn
 3 import torch.nn.functional as F
 4 import torch.optim as optim
 5 from torchvision import datasets, transforms
 6 
 7 from visdom import Visdom
 8 
 9 batch_size = 200
10 learning_rate = 0.01
11 epochs = 10
12 
13 train_loader = torch.utils.data.DataLoader(
14     datasets.MNIST('../data', train=True, download=True,
15                    transform=transforms.Compose([
16                        transforms.ToTensor(),
17                        # transforms.Normalize((0.1307,), (0.3081,))
18                    ])),
19     batch_size=batch_size, shuffle=True)
20 test_loader = torch.utils.data.DataLoader(
21     datasets.MNIST('../data', train=False, transform=transforms.Compose([
22         transforms.ToTensor(),
23         # transforms.Normalize((0.1307,), (0.3081,))
24     ])),
25     batch_size=batch_size, shuffle=True)
26 
27 
28 class MLP(nn.Module):
29     def __init__(self):
30         super(MLP, self).__init__()
31 
32         self.model = nn.Sequential(
33             nn.Linear(784, 200),
34             nn.LeakyReLU(inplace=True),
35             nn.Linear(200, 200),
36             nn.LeakyReLU(inplace=True),
37             nn.Linear(200, 10),
38             nn.LeakyReLU(inplace=True),
39         )
40 
41     def forward(self, x):
42         x = self.model(x)
43         return x
44 
45 
46 net = MLP()
47 optimizer = optim.SGD(net.parameters(), lr=learning_rate)
48 criteon = nn.CrossEntropyLoss()
49 
50 viz = Visdom()
51 
52 viz.line([0.], [0.], win='train_loss', opts=dict(title='train loss'))
53 viz.line([[0.0, 0.0]], [0.], win='test', opts=dict(title='test loss&acc.',
54                                                    legend=['loss', 'acc.']))
55 global_step = 0
56 
57 for epoch in range(epochs):
58 
59     for batch_idx, (data, target) in enumerate(train_loader):
60         data = data.view(-1, 28 * 28)
61 
62         logits = net(data)
63         loss = criteon(logits, target)
64 
65         optimizer.zero_grad()
66         loss.backward()
67         # print(w1.grad.norm(), w2.grad.norm())
68         optimizer.step()
69 
70         global_step += 1
71         viz.line([loss.item()], [global_step], win='train_loss', update='append')
72 
73         if batch_idx % 100 == 0:
74             print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
75                 epoch, batch_idx * len(data), len(train_loader.dataset),
76                        100. * batch_idx / len(train_loader), loss.item()))
77 
78     test_loss = 0
79     correct = 0
80     for data, target in test_loader:
81         data = data.view(-1, 28 * 28)
82         logits = net(data)
83         test_loss += criteon(logits, target).item()
84 
85         pred = logits.argmax(dim=1)
86         correct += pred.eq(target).float().sum().item()
87 
88     viz.line([[test_loss, correct / len(test_loader.dataset)]],
89              [global_step], win='test', update='append')
90     viz.images(data.view(-1, 1, 28, 28), win='x')
91     viz.text(str(pred.detach().cpu().numpy()), win='pred',
92              opts=dict(title='pred'))
93 
94     test_loss /= len(test_loader.dataset)
95     print('\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n'.format(
96         test_loss, correct, len(test_loader.dataset),
97         100. * correct / len(test_loader.dataset)))

View Code

posted @ 2020-07-27 09:10 小吴的日常阅读(154) 评论(0) 收藏举报

刷新页面返回顶部

wupiao

pytorch-day04

1、梯度

2、常见函数梯度

3、激活函数梯度

4、LOSS及其梯度

5、感知机的梯度推导

6、链式法则

7、MLP（多层感知机）反向传播

8、Logistic Regression

9、交叉熵（不确定性）

10、LR多分类实战（底层实现）

11、全连接层（使用API实现）

12、激活函数与GPU加速

13、MNIST测试

14、Visdom可视化（pip install visdom）

公告

wupiao

pytorch-day04

1、梯度

2、常见函数梯度

3、激活函数梯度

4、LOSS及其梯度

5、感知机的梯度推导

6、 链式法则

7、MLP（多层感知机）反向传播

8、Logistic Regression

9、交叉熵（不确定性）

10、LR多分类实战（底层实现）

11、全连接层（使用API实现）

12、激活函数与GPU加速

13、MNIST测试

14、Visdom可视化（pip install visdom）

公告

6、链式法则