pytorch-day10（AutoEncoder&VAE）

 1、AutoEncoder

　Why needed Unsupervised Learning?（如何利用好这些无监督数据，从重建自己的过程中，发现数据的模态）

 　　

　AutoEncoder：

　

　How to Train?

　　

　PCA VS Auto-Encoders：

　　

2、AutoEncoder 变体

　2.1、Denoising AutoEncoders

　　

　2.2、Dropout AutoEncoders

　　

---

 

　2.3、Adversarial AutoEncoders

 　　

　　method 1 ：

　　　　　　

　　method 2：

　　　　

　　

　KL：

　　

 　　　

　正态分布的情况：

　　　

　q~N(0,1)时，实现如下：

　　　　　　

　问题：Sample() is not differentiable

　解决：

　

　　

　　

　Generative model：

　　

　　

---

3、AE的实现

 　ae.py：

import torch
from torch import nn


class AE(nn.Module):
    def __init__(self):
        super(AE, self).__init__()

        # [b, 784] => [b, 20]
        self.encoder = nn.Sequential(
            nn.Linear(784, 256),
            nn.ReLU(),
            nn.Linear(256, 64),
            nn.ReLU(),
            nn.Linear(64, 20),
            nn.ReLU()
        )
        # [b, 20] => [b, 784]
        self.decoder = nn.Sequential(
            nn.Linear(20, 64),
            nn.ReLU(),
            nn.Linear(64, 256),
            nn.ReLU(),
            nn.Linear(256, 784),
            nn.Sigmoid()  # 把像素压缩到0-1
        )

    def forward(self, x):
        batchsz = x.size(0)
        # flatten
        x = x.view(batchsz, 784)
        # encoder
        x = self.encoder(x)
        # decoder
        x = self.decoder(x)
        # reshape
        x = x.view(batchsz, 1, 28, 28)

        return x, None

　main.py：

import torch
from torch.utils.data import DataLoader
from torch import nn, optim
from torchvision import transforms, datasets
import visdom
from ae import AE


def main():
    mnist_train = datasets.MNIST('mnist', True, transform=transforms.Compose([
        transforms.ToTensor()
    ]), download=True)
    mnist_train = DataLoader(mnist_train, batch_size=32, shuffle=True)

    mnist_test = datasets.MNIST('mnist', False, transform=transforms.Compose([
        transforms.ToTensor()
    ]), download=True)
    mnist_test = DataLoader(mnist_test, batch_size=32, shuffle=True)

    # x, _ = iter(mnist_train).next()  # 这里不需要label
    # print('x:', x.shape)  # x: torch.Size([32, 1, 28, 28])

    viz = visdom.Visdom()

    model = AE()
    criteon = nn.MSELoss()
    optimizer = optim.Adam(model.parameters(), lr=1e-3)
    print(model)

    for epoch in range(1000):
        for batchidx, (x, _) in enumerate(mnist_train):  # mnist_train
            # [b, 1, 28, 28]

            x_hat, kld = model(x)
            loss = criteon(x_hat, x)

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

        print(epoch, 'loss:', loss.item())

        x, _ = iter(mnist_test).next()  # mnist_test
        with torch.no_grad():
            x_hat, kld = model(x)
        viz.images(x, nrow=8, win='x', opts=dict(title='x'))
        viz.images(x_hat, nrow=8, win='x_hat', opts=dict(title='x_hat'))


if __name__ == '__main__':
    main()

 4、VAE的实现

 　vae.py：

import torch
from torch import nn


class VAE(nn.Module):

    def __init__(self):
        super(VAE, self).__init__()

        # [b, 784] => [b, 20]
        # u: [b, 10]
        # sigma: [b, 10]
        self.encoder = nn.Sequential(
            nn.Linear(784, 256),
            nn.ReLU(),
            nn.Linear(256, 64),
            nn.ReLU(),
            nn.Linear(64, 20),
            nn.ReLU()
        )
        # [b, 20] => [b, 784]
        self.decoder = nn.Sequential(
            nn.Linear(10, 64),  # 与encoder的最后一层并不对应
            nn.ReLU(),
            nn.Linear(64, 256),
            nn.ReLU(),
            nn.Linear(256, 784),
            nn.Sigmoid()
        )

        self.criteon = nn.MSELoss()

    def forward(self, x):
        batchsz = x.size(0)
        # flatten
        x = x.view(batchsz, 784)
        # encoder
        # [b, 20], including mean and sigma
        h_ = self.encoder(x)
        # [b, 20] => [b, 10] and [b, 10]
        mu, sigma = h_.chunk(2, dim=1)  # 张量分块，返回一个张量列表,按列分成2块
        print(mu.size(),sigma.size())
        # reparametrize trick, epison~N(0, 1) 解决sample不可导问题
        h = mu + sigma * torch.randn_like(sigma)  # torch.randn_like():生成一个shape相同的随机矩阵 *:均值为0、方差为1的正态分布

        # decoder
        x_hat = self.decoder(h)
        # reshape
        x_hat = x_hat.view(batchsz, 1, 28, 28)

        kld = 0.5 * torch.sum(  # 在网上找公式来实现就行(q~N(0,1)时)
            torch.pow(mu, 2) +
            torch.pow(sigma, 2) -
            torch.log(1e-8 + torch.pow(sigma, 2)) - 1
        ) / (batchsz * 28 * 28)

        return x_hat, kld

　main.py：

import torch
from torch.utils.data import DataLoader
from torch import nn, optim
from torchvision import transforms, datasets
import visdom
from vae import VAE


def main():
    mnist_train = datasets.MNIST('mnist', True, transform=transforms.Compose([
        transforms.ToTensor()
    ]), download=True)
    mnist_train = DataLoader(mnist_train, batch_size=32, shuffle=True)

    mnist_test = datasets.MNIST('mnist', False, transform=transforms.Compose([
        transforms.ToTensor()
    ]), download=True)
    mnist_test = DataLoader(mnist_test, batch_size=32, shuffle=True)

    # x, _ = iter(mnist_train).next()
    # print('x:', x.shape)

    viz = visdom.Visdom()

    # device = torch.device('cuda')
    model = VAE()
    criteon = nn.MSELoss()
    optimizer = optim.Adam(model.parameters(), lr=1e-3)
    print(model)

    for epoch in range(1000):
        for batchidx, (x, _) in enumerate(mnist_train):
            # [b, 1, 28, 28]

            x_hat, kld = model(x)
            loss = criteon(x_hat, x)

            if kld is not None:
                elbo = - loss - 1.0 * kld
                loss = - elbo

            # backprop
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()

        print(epoch, 'loss:', loss.item(), 'kld:', kld.item())

        x, _ = iter(mnist_test).next()
        with torch.no_grad():
            x_hat, kld = model(x)
        viz.images(x, nrow=8, win='x', opts=dict(title='x'))
        viz.images(x_hat, nrow=8, win='x_hat', opts=dict(title='x_hat'))


if __name__ == '__main__':
    main()

---