vgg16网络及pytorch神经网络

一、基于tensorflow的vgg16：识别猫狗数据集

import os, shutil
current_dir = (r"E:\人工智能\猫狗数据集\dogs-vs-cats")  # 当前目录
current_dir[0]
base_dir = current_dir[0] + ':/人工智能/cats_dogs_small'
os.mkdir(base_dir)  # 创建目录
# 分别创建训练集、验证集和测试集的目录

train_dir = os.path.join(base_dir,"train")
os.mkdir(train_dir)
validation_dir = os.path.join(base_dir,"validation")
os.mkdir(validation_dir)
test_dir = os.path.join(base_dir,"test")
os.mkdir(test_dir)

# 猫、狗的训练、验证、测试图像目录
train_cats_dir = os.path.join(train_dir, "cats")
os.mkdir(train_cats_dir)
train_dogs_dir = os.path.join(train_dir, "dogs")
os.mkdir(train_dogs_dir)

validation_cats_dir = os.path.join(validation_dir, "cats")
os.mkdir(validation_cats_dir)
validation_dogs_dir = os.path.join(validation_dir, "dogs")
os.mkdir(validation_dogs_dir)

test_cats_dir = os.path.join(test_dir, "cats")
os.mkdir(test_cats_dir)
test_dogs_dir = os.path.join(test_dir, "dogs")
os.mkdir(test_dogs_dir)

# 1000张当做训练集train
fnames = ['cat.{}.jpg'.format(i) for i in range(1000)]
for fname in fnames:
    # 源目录文件
    src = os.path.join(current_dir[0] + ":/人工智能/猫狗数据集/dogs-vs-cats/train", fname)
    # 目标目录
    dst = os.path.join(train_cats_dir, fname)
    shutil.copyfile(src, dst)
    
# 500张当做验证集valiation
fnames = ['cat.{}.jpg'.format(i) for i in range(1000,1500)]
for fname in fnames:
    src = os.path.join(current_dir[0] + ":人工智能/猫狗数据集/dogs-vs-cats/train", fname)
    dst = os.path.join(validation_cats_dir, fname)
    shutil.copyfile(src, dst)
    
# 500张当做测试集test
fnames = ['cat.{}.jpg'.format(i) for i in range(1500,2000)]
for fname in fnames:
    src = os.path.join(current_dir[0] + ":/人工智能/猫狗数据集/dogs-vs-cats/train", fname)
    dst = os.path.join(test_cats_dir, fname)
    shutil.copyfile(src, dst)

# 针对dog的3个同样操作

# 1、1000张当做训练集train
fnames = ['dog.{}.jpg'.format(i) for i in range(1000)]
for fname in fnames:
    src = os.path.join(current_dir[0] + ":/人工智能/猫狗数据集/dogs-vs-cats/train", fname)
    dst = os.path.join(train_dogs_dir, fname)
    shutil.copyfile(src, dst)
    
# 2、500张当做验证集valiation
fnames = ['dog.{}.jpg'.format(i) for i in range(1000,1500)]
for fname in fnames:
    src = os.path.join(current_dir[0] + ":/人工智能/猫狗数据集/dogs-vs-cats/train", fname)
    dst = os.path.join(validation_dogs_dir, fname)
    shutil.copyfile(src, dst)
    
# 3、500张当做测试集test
fnames = ['dog.{}.jpg'.format(i) for i in range(1500,2000)]
for fname in fnames:
    src = os.path.join(current_dir[0] + ":/人工智能/猫狗数据集/dogs-vs-cats/train", fname)
    dst = os.path.join(test_dogs_dir, fname)
    shutil.copyfile(src, dst)

#构建神经网络
import tensorflow as tf
from keras import layers 
from keras import models

model = models.Sequential()
model.add(tf.keras.layers.Conv2D(32,(3,3),activation="relu",
                               input_shape=(150,150,3)))
model.add(tf.keras.layers.MaxPooling2D((2,2)))

model.add(tf.keras.layers.Conv2D(64,(3,3),activation="relu"))
model.add(tf.keras.layers.MaxPooling2D((2,2)))

model.add(tf.keras.layers.Conv2D(128,(3,3),activation="relu"))
model.add(tf.keras.layers.MaxPooling2D((2,2)))

model.add(tf.keras.layers.Conv2D(128,(3,3),activation="relu"))
model.add(tf.keras.layers.MaxPooling2D((2,2)))  

model.add(tf.keras.layers.Flatten())

model.add(tf.keras.layers.Dropout(0.5))

model.add(tf.keras.layers.Dense(512, activation="relu"))
model.add(tf.keras.layers.Dense(1, activation="sigmoid"))

model.compile(loss="binary_crossentropy",
             optimizer=optimizers.RMSprop(lr=1e-4),
             metrics=["acc"])

model.summary()

 

from tensorflow.keras import optimizers

model.compile(loss="binary_crossentropy",
             optimizer=optimizers.RMSprop(lr=1e-4),
             metrics=["acc"])
def generator():
    i=0
    while True:
        i += 1
        yield i
for item in generator():
    print(item)
    if item>4:
        break

from keras.preprocessing.image import ImageDataGenerator

train_datagen = ImageDataGenerator(rescale=1./255)  # 进行缩放
test_datagen = ImageDataGenerator(rescale=1./255)  # 进行缩放

train_generator = train_datagen.flow_from_directory(
    train_dir,  # 待处理的目录
    target_size=(150,150),  # 图像大小设置
    batch_size=20,
    class_mode="binary"  # 损失函数是binary_crossentropy 所以使用二进制标签
)

validation_generator = test_datagen.flow_from_directory(
    validation_dir,  # 待处理的目录
    target_size=(150,150),  # 图像大小设置
    batch_size=20,
    class_mode="binary"  # 损失函数是binary_crossentropy 所以使用二进制标签
)

for data_batch, labels_batch in train_generator:
    print(data_batch.shape)
    print(labels_batch.shape)
    break

 

 

history = model.fit_generator(
    train_generator,  # 第一个参数必须是Python生成器
    steps_per_epoch=100,  # 2000 / 20
    epochs=10,  # 迭代次数
    validation_data=validation_generator,  # 待验证的数据集
    validation_steps=50 
)

 

# 保存模型
model.save("cats_and_dogs_small.h5")
import matplotlib.pyplot as plt
%matplotlib inline
history_dict = history.history  # 字典形式
for key, _ in history_dict.items():
    print(key)

acc = history_dict["acc"]
val_acc = history_dict["val_acc"]

loss = history_dict["loss"]
val_loss = history_dict["val_loss"]

 

 

acc = history_dict["acc"]
val_acc = history_dict["val_acc"]

loss = history_dict["loss"]
val_loss = history_dict["val_loss"]

epochs = range(1, len(acc)+1)

# acc
plt.plot(epochs, acc, "bo", label="Training acc")
plt.plot(epochs, val_acc, "b", label="Validation acc")
plt.title("Training and Validation acc")
plt.legend()

plt.figure()

# loss
plt.plot(epochs, loss, "bo", label="Training loss")
plt.plot(epochs, val_loss, "b", label="Validation loss")
plt.title("Training and Validation loss")
plt.legend()

 

 

#设置数据增强
datagen = ImageDataGenerator(
    rotation_range=40,  # 0-180的角度值
    width_shift_range=0.2,  # 水平和垂直方向的范围；相对于总宽度或者高度的比例
    height_shift_range=0.2,
    shear_range=0.2,  # 随机错切变换的角度
    zoom_range=0.2,  # 图像随机缩放的角度
    horizontal_flip=True,  # 随机将一半图像进行水平翻转
    fill_mode="nearest"  # 用于填充新创建像素的方法
)
#显示增强后图像
from keras.preprocessing import image
fnames = [os.path.join(train_cats_dir,fname) for fname in os.listdir(train_cats_dir)]
img_path = fnames[3]
# 读取图片并调整大小
img = image.load_img(img_path, target_size=(150,150))  
# 转成数组
x = image.img_to_array(img)

# shape转成(1,150,150,3)
x = x.reshape((1,) + x.shape)

i = 0
for batch in datagen.flow(x, batch_size=1):  # 生成随机变换后的图像批量
    plt.figure()   
    imgplot = plt.imshow(image.array_to_img(batch[0]))
    i += 1
    if i % 4 == 0:
        break  # 循环是无限，需要在某个时刻终止      
plt.show()

 

 

# 训练数据的增强
train_datagen = ImageDataGenerator(
    rescale=1. / 255,
    rotation_range=40,
    width_shift_range=0.2,
    height_shift_range=0.2,
    shear_range=0.2,
    zoom_range=0.2,
    horizontal_flip=True
)

# 不能增强验证数据
test_datagen = ImageDataGenerator(rescale=1.0 / 255)

train_generator = train_datagen.flow_from_directory(
    train_dir,  # 目标目录
    target_size=(150,150),  # 大小调整
    batch_size=32,
    class_mode="binary"
)

validation_generator = test_datagen.flow_from_directory(
    validation_dir,
    target_size=(150,150),
    batch_size=32,
    class_mode="binary"
)

# 优化：报错有修改
history = model.fit_generator(
    train_generator,
    # 原文 steps_per_epoch=100,
    steps_per_epoch=63,  # steps_per_epoch=2000/32≈63 取上限
    epochs=100,
    validation_data=validation_generator,
    # 原文 validation_steps=50
    validation_steps=32  # validation_steps=1000/32≈32
)

 

 

# 保存模型
model.save("cats_and_dogs_small_2.h5")

history_dict = history.history  # 字典形式

acc = history_dict["acc"]
val_acc = history_dict["val_acc"]
loss = history_dict["loss"]
val_loss = history_dict["val_loss"]

epochs = range(1, len(acc)+1)

# acc
plt.plot(epochs, acc, "bo", label="Training acc")
plt.plot(epochs, val_acc, "b", label="Validation acc")
plt.title("Training and Validation acc")
plt.legend()

plt.figure()

# loss
plt.plot(epochs, loss, "bo", label="Training loss")
plt.plot(epochs, val_loss, "b", label="Validation loss")
plt.title("Training and Validation loss")
plt.legend()

plt.show()

 

 

 二、pytorch神经网络

#定义神经网络
import torch
import torch.nn as nn
import torch.nn.functional as F


class Net(nn.Module):

    def __init__(self):
        super(Net, self).__init__()
        # 1 input image channel, 6 output channels, 5x5 square convolution
        # kernel
        self.conv1 = nn.Conv2d(1, 6, 5)
        self.conv2 = nn.Conv2d(6, 16, 5)
        # an affine operation: y = Wx + b
        self.fc1 = nn.Linear(16 * 5 * 5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)

    def forward(self, x):
        # Max pooling over a (2, 2) window
        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
        # If the size is a square you can only specify a single number
        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
        x = x.view(-1, self.num_flat_features(x))
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

    def num_flat_features(self, x):
        size = x.size()[1:]  # all dimensions except the batch dimension
        num_features = 1
        for s in size:
            num_features *= s
        return num_features


net = Net()
print(net)

 

#一个模型可训练的参数可以通过调用 net.parameters() 返回
params = list(net.parameters())
print(len(params))
print(params[0].size())  # conv1's .weight

 

#尝试随机生成一个 32x32 的输入
input = torch.randn(1, 1, 32, 32)
out = net(input)
print(out)

 

 

#把所有参数梯度缓存器置零，用随机的梯度来反向传播
net.zero_grad()
out.backward(torch.randn(1, 10))
#损失函数
output = net(input)
target = torch.randn(10)  # a dummy target, for example
target = target.view(1, -1)  # make it the same shape as output
criterion = nn.MSELoss()

loss = criterion(output, target)
print(loss)

 

 

print(loss.grad_fn)  # MSELoss
print(loss.grad_fn.next_functions[0][0])  # Linear
print(loss.grad_fn.next_functions[0][0].next_functions[0][0])  # ReLU

 

 

#反向传播
net.zero_grad()     # zeroes the gradient buffers of all parameters

print('conv1.bias.grad before backward')
print(net.conv1.bias.grad)

loss.backward()

print('conv1.bias.grad after backward')
print(net.conv1.bias.grad)

#更新神经网络的参数
#weight = weight - learning_rate * gradient  随机梯度下降
learning_rate = 0.01
for f in net.parameters():
    f.data.sub_(f.grad.data * learning_rate)
    
import torch.optim as optim

# create your optimizer
optimizer = optim.SGD(net.parameters(), lr=0.01)

# in your training loop:
optimizer.zero_grad()   # zero the gradient buffers
output = net(input)
loss = criterion(output, target)
loss.backward()
optimizer.step()    # Does the update