猫狗识别的keras实现

发布于 2021-09-07  5956 次阅读


数据预处理

首先将 train中的照片分别划分为猫狗的测试集、验证集和训练集 。(原始从数据集可在如下网址下载:https://www.kaggle.com/c/dogs-vs-cats/data

#对原始数据集train中的照片分别划分为猫狗的测试集、验证集和训练集
import os,shutil

original_dataset_dir = r'E:\学习资料\AI\data\dogs-vs-cats\train'#原始数据集解压目录的路径
base_dir = r'E:\学习资料\AI\data\dogs-vs-cats\cats_and_dogs_datas_small'
os.mkdir(base_dir)#创建文件夹

#创建数据集总目录
train_dir = os.path.join(base_dir,'训练集')
vaildation_dir = os.path.join(base_dir,'验证集')
test_dir = os.path.join(base_dir,'测试集')
os.mkdir(train_dir)
os.mkdir(vaildation_dir)
os.mkdir(test_dir)

#创建猫狗的训练集数据
train_cats_dir = os.path.join(train_dir,'cats')
train_dogs_dir = os.path.join(train_dir,'dogs')
os.mkdir(train_cats_dir)
os.mkdir(train_dogs_dir)

#创建猫狗的验证集数据
vaildation_cats_dir = os.path.join(vaildation_dir,'cats')
vaildation_dogs_dir = os.path.join(vaildation_dir,'dogs')
os.mkdir(vaildation_cats_dir)
os.mkdir(vaildation_dogs_dir)

#创建猫狗的测试集数据
test_cats_dir = os.path.join(test_dir,'cats')
test_dogs_dir = os.path.join(test_dir,'dogs')
os.mkdir(test_cats_dir)
os.mkdir(test_dogs_dir)

#将前1000张图片复制到train_cats_dir文件夹中
fnames = ['cat.{}.jpg'.format(i) for i in range(1000)]
for fname in fnames:
    src = os.path.join(original_dataset_dir,fname)
    dst = os.path.join(train_cats_dir,fname)
    shutil.copyfile(src,dst)

#将接下来500张图片复制到vaildation_cats_dir文件夹中
fnames = ['cat.{}.jpg'.format(i) for i in range(1000,1500)]
for fname in fnames:
    src = os.path.join(original_dataset_dir,fname)
    dst = os.path.join(vaildation_cats_dir,fname)
    shutil.copyfile(src,dst)

#将剩下500张图片复制到test_cats_dir文件夹中
fnames = ['cat.{}.jpg'.format(i) for i in range(1500,2000)]
for fname in fnames:
    src = os.path.join(original_dataset_dir,fname)
    dst = os.path.join(test_cats_dir,fname)
    shutil.copyfile(src,dst)

#将前1000张图片复制到train_dogs_dir文件夹中
fnames = ['dog.{}.jpg'.format(i) for i in range(1000)]
for fname in fnames:
    src = os.path.join(original_dataset_dir,fname)
    dst = os.path.join(train_dogs_dir,fname)
    shutil.copyfile(src,dst)

#将接下来500张图片复制到vaildation_dogs_dir文件夹中
fnames = ['dog.{}.jpg'.format(i) for i in range(1000,1500)]
for fname in fnames:
    src = os.path.join(original_dataset_dir,fname)
    dst = os.path.join(vaildation_dogs_dir,fname)
    shutil.copyfile(src,dst)

#将剩下500张图片复制到test_dogs_dir文件夹中
fnames = ['dog.{}.jpg'.format(i) for i in range(1500,2000)]
for fname in fnames:
    src = os.path.join(original_dataset_dir,fname)
    dst = os.path.join(test_dogs_dir,fname)
    shutil.copyfile(src,dst)

为了便于理解各个功能所作出的贡献,我们只取少量数据集用于学习

#查看各数据集数据大小
import os
train_cats_dir = 'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\训练集\\cats'
train_dogs_dir = 'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\训练集\\dogs'
vaildation_cats_dir = 'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\验证集\\cats'
vaildation_dogs_dir = 'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\验证集\\dogs'
test_cats_dir = 'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\测试集\\cats'
test_dogs_dir = 'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\测试集\\dogs'

print('猫的训练集图片:',len(os.listdir(train_cats_dir)))
print('狗的训练集图片:',len(os.listdir(train_dogs_dir)))
print('猫的验证集图片:',len(os.listdir(vaildation_cats_dir)))
print('狗的验证集图片:',len(os.listdir(vaildation_dogs_dir)))
print('猫的测试集图片:',len(os.listdir(test_cats_dir)))
print('狗的测试集图片:',len(os.listdir(test_dogs_dir)))
猫的训练集图片: 1000
狗的训练集图片: 1000
猫的验证集图片: 500
狗的验证集图片: 500
猫的测试集图片: 500
狗的测试集图片: 500
# 数据预处理
from tensorflow.keras.preprocessing.image import ImageDataGenerator

train_datagen = ImageDataGenerator(rescale = 1./255)
test_datagen = ImageDataGenerator(rescale = 1./255)

train_generator = train_datagen.flow_from_directory(
    'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\训练集',
    target_size = (150,150),
    batch_size = 20,
    class_mode = 'binary')

validation_generator = test_datagen.flow_from_directory(
    'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\验证集',
    target_size = (150,150),
    batch_size = 20,
    class_mode = 'binary')
Found 2000 images belonging to 2 classes.
Found 1000 images belonging to 2 classes.
#构建网络
from tensorflow.keras import layers
from tensorflow.keras import models

model = models.Sequential()
model.add(layers.Conv2D(32,(3,3),activation = 'relu',
                       input_shape = (150,150,3)))#32为通道数量,(3,3)为卷积块大小
model.add(layers.MaxPooling2D((2,2)))
model.add(layers.Conv2D(64,(3,3),activation = 'relu'))
model.add(layers.MaxPooling2D((2,2)))
model.add(layers.Conv2D(128,(3,3),activation = 'relu'))
model.add(layers.MaxPooling2D((2,2)))
model.add(layers.Conv2D(128,(3,3),activation = 'relu'))
model.add(layers.MaxPooling2D((2,2)))
model.add(layers.Flatten())#数据一维化
model.add(layers.Dense(512,activation = 'relu'))
model.add(layers.Dense(1,activation = 'sigmoid'))
model.summary()
Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_4 (Conv2D)            (None, 148, 148, 32)      896       
_________________________________________________________________
max_pooling2d_4 (MaxPooling2 (None, 74, 74, 32)        0         
_________________________________________________________________
conv2d_5 (Conv2D)            (None, 72, 72, 64)        18496     
_________________________________________________________________
max_pooling2d_5 (MaxPooling2 (None, 36, 36, 64)        0         
_________________________________________________________________
conv2d_6 (Conv2D)            (None, 34, 34, 128)       73856     
_________________________________________________________________
max_pooling2d_6 (MaxPooling2 (None, 17, 17, 128)       0         
_________________________________________________________________
conv2d_7 (Conv2D)            (None, 15, 15, 128)       147584    
_________________________________________________________________
max_pooling2d_7 (MaxPooling2 (None, 7, 7, 128)         0         
_________________________________________________________________
flatten_1 (Flatten)          (None, 6272)              0         
_________________________________________________________________
dense_2 (Dense)              (None, 512)               3211776   
_________________________________________________________________
dense_3 (Dense)              (None, 1)                 513       
=================================================================
Total params: 3,453,121
Trainable params: 3,453,121
Non-trainable params: 0
_________________________________________________________________
# 配置模型
from tensorflow.keras import optimizers

model.compile(loss = 'binary_crossentropy',
              optimizer = optimizers.RMSprop(lr=1e-4),
              metrics = ['acc'])

#训练模型
history = model.fit_generator(train_generator,
                             steps_per_epoch = 100,
                             epochs = 30,
                             validation_data = validation_generator,
                             validation_steps = 50)
#绘制损失曲线和精度曲线
import matplotlib.pyplot as plt

acc = history.history['acc']
loss = history.history['loss']
val_acc = history.history['val_acc']
val_loss = history.history['val_loss']
epochs = range(1,len(acc)+1)

plt.plot(epochs,acc,'bo',label = 'Training acc')
plt.plot(epochs,val_acc,'b',label = 'Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs,loss,'bo',label = 'Training loss')
plt.plot(epochs,val_loss,'b',label = 'Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()

可以看到,训练精度慢慢趋向于100%,而验证精度则一直在75%左右徘徊;训练损失接近线性下降,直到接近于0,但验证损失尽在7、8轮左右达到最小,之后快速增加。
这是因为训练样本相对较少(2000个),所以过拟合问题亟需解决,如使用dropout和权重衰减(L2正则化)。而我们这次将使用一种针对计算机视觉领域的新方法,在用深度学习模型处理图像时几乎都会用到这种方法,它就是数据增强(data augmentation)。

数据增强并添加dropou层后

数据增强是从现有的训练样本中生成更多的训练数据,其方法是利用多种能够生成可信图像的随即变换来增加(augment)样本。其目标是,模型在训练时不会两次查看完全相同的图像。这让模型能够观察到数据的更多内容,从而获得更好的泛化能力。 在keras中,可以通过ImageDataGenerator实例读取的图像执行多次随即变换来实现。

#数据增强
datagen = ImageDataGenerator(
    rotation_range = 40,#随机旋转的角度范围(0-180°)
    width_shift_range = 0.2,#图像在水平方向上平移的范围(总宽度的比例)
    height_shift_range = 0.2,#图像在垂直方向上平移的范围(总高度的比例)
    shear_range = 0.2,#随机错切变换的角度
    zoom_range = 0.2,#图像随机缩放的范围
    horizontal_flip = True,#随机将一半图像水平翻转。如果没有水平不对称的假设(比如真实世界),这种变换是有意义的。
    fill_mode = 'nearest')#用于填充新创建像素的方法,这些新像素可能来自于旋转或宽度/高度平移。

如果使用这种数据增强来训练一个新网络,那么网络将不会看到两次同样的输入。但网络看到的输入任然是高度相关的,因为这些输入都来自于少量的原始图像。你无法生成新信息,而只能混合现有信息。因此,这种方法可能不足以完全消除过拟合。 为了进一步降低过拟合,还需要向模型中添加一个Dropout层,添加到密集连接分类器之前。

#定义一个包含Dropout的新卷积神经网络
model = models.Sequential()
model.add(layers.Conv2D(32,(3,3),activation = 'relu',
                       input_shape = (150,150,3)))#32为通道数量,(3,3)为卷积块大小
model.add(layers.MaxPooling2D((2,2)))
model.add(layers.Conv2D(64,(3,3),activation = 'relu'))
model.add(layers.MaxPooling2D((2,2)))
model.add(layers.Conv2D(128,(3,3),activation = 'relu'))
model.add(layers.MaxPooling2D((2,2)))
model.add(layers.Conv2D(128,(3,3),activation = 'relu'))
model.add(layers.MaxPooling2D((2,2)))
model.add(layers.Flatten())#数据一维化
model.add(layers.Dropout(0.5))
model.add(layers.Dense(512,activation = 'relu'))
model.add(layers.Dense(1,activation = 'sigmoid'))
model.summary()

model.compile(loss = 'binary_crossentropy',
             optimizer = optimizers.RMSprop(lr = 1e-4),
             metrics = ['acc'])
Model: "sequential_2"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_8 (Conv2D)            (None, 148, 148, 32)      896       
_________________________________________________________________
max_pooling2d_8 (MaxPooling2 (None, 74, 74, 32)        0         
_________________________________________________________________
conv2d_9 (Conv2D)            (None, 72, 72, 64)        18496     
_________________________________________________________________
max_pooling2d_9 (MaxPooling2 (None, 36, 36, 64)        0         
_________________________________________________________________
conv2d_10 (Conv2D)           (None, 34, 34, 128)       73856     
_________________________________________________________________
max_pooling2d_10 (MaxPooling (None, 17, 17, 128)       0         
_________________________________________________________________
conv2d_11 (Conv2D)           (None, 15, 15, 128)       147584    
_________________________________________________________________
max_pooling2d_11 (MaxPooling (None, 7, 7, 128)         0         
_________________________________________________________________
flatten_2 (Flatten)          (None, 6272)              0         
_________________________________________________________________
dropout_2 (Dropout)          (None, 6272)              0         
_________________________________________________________________
dense_4 (Dense)              (None, 512)               3211776   
_________________________________________________________________
dense_5 (Dense)              (None, 1)                 513       
=================================================================
Total params: 3,453,121
Trainable params: 3,453,121
Non-trainable params: 0
_________________________________________________________________
#利用数据增强生成器训练卷积神经网络
train_datagen = ImageDataGenerator(rescale = 1./255,
                                  rotation_range = 40,
                                  width_shift_range = 0.2,
                                  height_shift_range = 0.2,
                                  shear_range = 0.2,
                                  zoom_range = 0.2,
                                  horizontal_flip = True,
                                  fill_mode = 'nearest')

test_datagen = ImageDataGenerator(rescale = 1./255)#注意,不能增强验证图像

train_generator = train_datagen.flow_from_directory(
    'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\训练集',
    target_size = (150,150),
    batch_size = 20,#修改数值32为20了,具体原因见https://www.cnblogs.com/Renyi-Fan/p/13795851.html
    class_mode = 'binary')

validation_generator = test_datagen.flow_from_directory(
    'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\验证集',
    target_size = (150,150),
    batch_size = 20,
    class_mode = 'binary')

history = model.fit_generator(train_generator,
                             steps_per_epoch = 100,
                             epochs = 100,
                             validation_data = validation_generator,
                             validation_steps = 50)

#绘制损失曲线和精度曲线
import matplotlib.pyplot as plt

acc = history.history['acc']
loss = history.history['loss']
val_acc = history.history['val_acc']
val_loss = history.history['val_loss']
epochs = range(1,len(acc)+1)

plt.plot(epochs,acc,'bo',label = 'Training acc')
plt.plot(epochs,val_acc,'b',label = 'Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs,loss,'bo',label = 'Training loss')
plt.plot(epochs,val_loss,'b',label = 'Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()

可以看到,使用了数据增强和dropout后,模型不再过拟合:训练曲线紧紧跟随着验证曲线。通过进一步使用正则化方法以及调节网络参数(比如每个卷积层的过滤器个数或网络中的层数),我们可以得到更高的精度。但只靠从头开始训练自己的卷积神经网络,再想提高精度就十分困难,因为可用的数据太少。要想在这个问题上进一步提高精度,就需要使用预训练的模型。

引入VGG16架构

想要将深度学习应用于小型图像数据集,一种常用切非常高效的方法时使用预训练网络。预训练网络(pretrained network)是一个保存好的网络,之前已在大型数据集(通常是大规模图像分类任务)上训练好。如果这个原始数据集足够大且通用,那么预训练网络学到的特征的空间层次结构可以有效地作为视觉世界的通用模型,因此这些特征可用于各种不同的计算机视觉问题,即使这些新问题涉及的类别和原始任务完全不同。

这种学到的特征在不同问题之间的可移植性,是深度学习与许多早期浅层学习方法相比的重要优势,它使得深度学习对小数据问题非常有效。

#使用VGG16架构
#将VGG16卷积基实例化
from tensorflow.keras.applications import VGG16

conv_base = VGG16(weights = 'imagenet',#指定模型初始化的权重检查点
                 include_top = False,#指定模型最后是否包含密集连接分类器。默认情况下,这个密集连接分类器对应于ImageNet的1000个剋别。因为我们打算使用自己的密集连接分类器(只有cat和dog两个类别),所以不需要使用它
                 input_shape = (150,150,3))#输入到网络中的图像张量的形状。参数可选,默认为能够处理任意形状的输入
conv_base.summary()
Model: "vgg16"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
input_4 (InputLayer)         [(None, 150, 150, 3)]     0         
_________________________________________________________________
block1_conv1 (Conv2D)        (None, 150, 150, 64)      1792      
_________________________________________________________________
block1_conv2 (Conv2D)        (None, 150, 150, 64)      36928     
_________________________________________________________________
block1_pool (MaxPooling2D)   (None, 75, 75, 64)        0         
_________________________________________________________________
block2_conv1 (Conv2D)        (None, 75, 75, 128)       73856     
_________________________________________________________________
block2_conv2 (Conv2D)        (None, 75, 75, 128)       147584    
_________________________________________________________________
block2_pool (MaxPooling2D)   (None, 37, 37, 128)       0         
_________________________________________________________________
block3_conv1 (Conv2D)        (None, 37, 37, 256)       295168    
_________________________________________________________________
block3_conv2 (Conv2D)        (None, 37, 37, 256)       590080    
_________________________________________________________________
block3_conv3 (Conv2D)        (None, 37, 37, 256)       590080    
_________________________________________________________________
block3_pool (MaxPooling2D)   (None, 18, 18, 256)       0         
_________________________________________________________________
block4_conv1 (Conv2D)        (None, 18, 18, 512)       1180160   
_________________________________________________________________
block4_conv2 (Conv2D)        (None, 18, 18, 512)       2359808   
_________________________________________________________________
block4_conv3 (Conv2D)        (None, 18, 18, 512)       2359808   
_________________________________________________________________
block4_pool (MaxPooling2D)   (None, 9, 9, 512)         0         
_________________________________________________________________
block5_conv1 (Conv2D)        (None, 9, 9, 512)         2359808   
_________________________________________________________________
block5_conv2 (Conv2D)        (None, 9, 9, 512)         2359808   
_________________________________________________________________
block5_conv3 (Conv2D)        (None, 9, 9, 512)         2359808   
_________________________________________________________________
block5_pool (MaxPooling2D)   (None, 4, 4, 512)         0         
=================================================================
Total params: 14,714,688
Trainable params: 14,714,688
Non-trainable params: 0
_________________________________________________________________

最后的特征图形状为(4,4,512)。我们将在这个特征上添加一个密集连接分类器。下一步有两种方法可供选择:
1.在数据集上运行卷积基,将输出保存成硬盘中的Numpy数组,然后用这个数据作为输入,输入到独立的密集连接分类器中。这种方法速度快,计算代价低,应为对于每个输入图像只需运行一次卷积基,而卷积基是目前流程中计算代价最高的。但出于同样的原因,这种方法不允许使用数据增强。
2.在顶部添加Dense层来扩展已有模型(即conv_base),并在输入数据上端到端地运行整个模型。这样就可以使用数据增强,因为每个输入图像进入模型时都会经过卷积基。但出于同样的原因,这种方法的计算代价比第一种要高很多。

不使用数据增强的快速特征提取

#不使用数据增强的快速特征提取
import os
import numpy as np
from tensorflow.keras.preprocessing.image import ImageDataGenerator

base_dir = 'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small'
train_dir = os.path.join(base_dir,'训练集')
validation_dir = os.path.join(base_dir,'验证集')
test_dir = os.path.join(base_dir,'测试集')

datagen = ImageDataGenerator(rescale = 1./255)
batch_size = 20

def extract_features(directory,sample_count):
    features = np.zeros(shape = (sample_count,4,4,512))
    labels = np.zeros(shape = (sample_count))
    generator = datagen.flow_from_directory(directory,
                                           target_size = (150,150),
                                           batch_size = batch_size,
                                           class_mode = 'binary')
    i = 0
    for inputs_batch,labels_batch in generator:
        features_batch = conv_base.predict(inputs_batch)
        features[i*batch_size : (i+1)*batch_size] = features_batch
        labels[i*batch_size : (i+1)*batch_size] = labels_batch
        i += 1
        if i *batch_size >= sample_count:
            break
    return features,labels

train_features ,train_labels = extract_features(train_dir,2000)
validation_features ,validation_labels = extract_features(validation_dir,1000)
test_features ,test_labels = extract_features(test_dir,1000)      

#目前提取到的特征形状为(samples,4,4,512)
#我们要将其输入到密集连接分类器中,所以首先必须将其形状展平为(samples,8192)
train_features = np.reshape(train_features,(2000,4*4*512))
validation_features = np.reshape(validation_features,(1000,4*4*512))
test_features = np.reshape(test_features,(1000,4*4*512))
Found 2000 images belonging to 2 classes.
Found 1000 images belonging to 2 classes.
Found 1000 images belonging to 2 classes.
#定义密集连接分类器(注意要使用dropout正则化),并在刚刚保存的数据和标签上训练这个分类器
from tensorflow.keras import layers
from tensorflow.keras import models
from tensorflow.keras import optimizers

model = models.Sequential()
model.add(layers.Dense(256,activation = 'relu',input_dim = 4*4*512))
model.add(layers.Dropout(0.5))
model.add(layers.Dense(1,activation = 'sigmoid'))

model.compile(optimizer = optimizers.RMSprop(lr=2e-5),
             loss = 'binary_crossentropy',
             metrics = ['acc'])

history = model.fit(train_features,train_labels,
                   epochs = 30,
                   batch_size = 20,
                   validation_data = (validation_features,validation_labels))

# 绘制损失曲线和精度曲线
import matplotlib.pyplot as plt

acc = history.history['acc']
loss = history.history['loss']
val_acc = history.history['val_acc']
val_loss = history.history['val_loss']
epochs = range(1,len(acc)+1)

plt.plot(epochs,acc,'bo',label = 'Training acc')
plt.plot(epochs,val_acc,'b',label = 'Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs,loss,'bo',label = 'Training loss')
plt.plot(epochs,val_loss,'b',label = 'Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()

验证精度达到了约90%,比之前的85左右又增加了一大步。但从图中可以看出,虽然dropout比率相当大,但模型几乎始终在过拟合。这是因为本方法没有使用数据增强,而数据增强对防止小型图像数据集的过拟合非常重要。

使用数据增强的特征提取

# 使用数据增强的特征提取
# 扩展conv_base模型,然后在输入数据上端到端地运行模型。
# 模型的行为和层类似,所以可以向Sequential模型中添加一个模型(比如conv_base),就像添加一个层一样。
from tensorflow.keras import layers
from tensorflow.keras import models

model = models.Sequential()
model.add(conv_base)
model.add(layers.Flatten())
model.add(layers.Dense(256,activation = 'relu'))
model.add(layers.Dense(1,activation = 'sigmoid'))
model.summary()
Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
vgg16 (Functional)           (None, 4, 4, 512)         14714688  
_________________________________________________________________
flatten (Flatten)            (None, 8192)              0         
_________________________________________________________________
dense (Dense)                (None, 256)               2097408   
_________________________________________________________________
dense_1 (Dense)              (None, 1)                 257       
=================================================================
Total params: 16,812,353
Trainable params: 16,812,353
Non-trainable params: 0
_________________________________________________________________

在编译和训练模型之前,一定要‘冻结’卷积基。冻结(freeze)一个或多个层是指在训练过程中保持其权重不变。如果不这么做,那么卷积基之前学到的表示将会在训练过程中被修改。因为器上添加的Dense层是随机初始化的,所以非常大的权重更新将会在网络上传播,对之前学到的表示造成很大破坏。
在keras中,冻结网络的方法就是将其trainable属性设为False。

#利用冻结的卷积基端到端地训练模型
from tensorflow.keras import optimizers
from tensorflow.keras.preprocessing.image import ImageDataGenerator

train_datagen = ImageDataGenerator(rescale = 1./255,
                                  rotation_range = 40,
                                  width_shift_range = 0.2,
                                  height_shift_range = 0.2,
                                  shear_range = 0.2,
                                  #zoom_range = 0.2,
                                  horizontal_flip = True,
                                  fill_mode = 'nearest')

test_datagen = ImageDataGenerator(rescale = 1./255)#注意,不能增强验证图像

train_generator = train_datagen.flow_from_directory(
    'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\训练集',
    target_size = (150,150),
    batch_size = 20,#修改数值32为20了,具体原因见https://www.cnblogs.com/Renyi-Fan/p/13795851.html
    class_mode = 'binary')

validation_generator = test_datagen.flow_from_directory(
    'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\验证集',
    target_size = (150,150),
    batch_size = 20,
    class_mode = 'binary')
Found 2000 images belonging to 2 classes.
Found 1000 images belonging to 2 classes.
model.compile(loss = 'binary_crossentropy',
             optimizer = optimizers.RMSprop(lr = 2e-5),
             metrics = ['acc'])

history = model.fit_generator(train_generator,
                             steps_per_epoch = 100,
                             epochs = 30,
                             validation_data = validation_generator,
                             validation_steps = 50)

#绘制损失曲线和精度曲线
import matplotlib.pyplot as plt

acc = history.history['acc']
loss = history.history['loss']
val_acc = history.history['val_acc']
val_loss = history.history['val_loss']
epochs = range(1,len(acc)+1)

plt.plot(epochs,acc,'bo',label = 'Training acc')
plt.plot(epochs,val_acc,'b',label = 'Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs,loss,'bo',label = 'Training loss')
plt.plot(epochs,val_loss,'b',label = 'Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()

微调模型

from tensorflow.keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras.applications import VGG16
from tensorflow.keras import optimizers
from tensorflow.keras import models
from tensorflow.keras import layers

# 数据预处理
train_dir = 'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\训练集'
validation_dir = 'E:\\学习资料\\AI\\data\\dogs-vs-cats\\cats_and_dogs_datas_small\\验证集'
train_datagen = ImageDataGenerator(
    rescale = 1./255,
    rotation_range = 40,
    width_shift_range = 0.2,
    height_shift_range = 0.2,
    shear_range = 0.2,
    horizontal_flip = True,
    fill_mode = 'nearest')
 
test_datagen = ImageDataGenerator(rescale = 1./255)
 
train_generator = train_datagen.flow_from_directory(
    train_dir,
    target_size = (150, 150),
    batch_size = 20,
    class_mode = 'binary')
 
validation_generator = test_datagen.flow_from_directory(
    validation_dir,
    target_size = (150, 150),
    batch_size = 20,
    class_mode = 'binary')
 
# 加载VGG16架构
conv_base2 = VGG16(weights = 'imagenet', 
                  include_top = False, 
                  input_shape = (150, 150, 3))

model2 = models.Sequential()
model2.add(conv_base2)
model2.add(layers.Flatten())
model2.add(layers.Dense(256, activation = 'relu'))
model2.add(layers.Dropout(0.5))
model2.add(layers.Dense(1, activation = 'sigmoid'))

conv_base2.trainable = True

set_trainable = False
for layer in conv_base2.layers:
    if layer.name == 'black5_conv1':
        set_trainable = True
    if set_trainable:
        layers.trainable = True
    else:
        layers.trainable = False
        
model2.compile(loss = 'binary_crossentropy',
             optimizer = optimizers.RMSprop(lr=1e-5),
             metrics = ['acc'])
 
history = model2.fit_generator(
    train_generator,
    steps_per_epoch = 100,
    epochs = 100,
    validation_data = validation_generator,
    validation_steps= 50)

# 保存模型
model.save('cats_and_dogs_small_vgg16_WeiTiao_droporp_4.h5')

# 绘制损失曲线和精度曲线
import matplotlib.pyplot as plt

acc = history.history['acc']
loss = history.history['loss']
val_acc = history.history['val_acc']
val_loss = history.history['val_loss']
epochs = range(1,len(acc)+1)

plt.plot(epochs,acc,'bo',label = 'Training acc')
plt.plot(epochs,val_acc,'b',label = 'Validation acc')
plt.title('Training and validation accuracy')
plt.legend()

plt.figure()

plt.plot(epochs,loss,'bo',label = 'Training loss')
plt.plot(epochs,val_loss,'b',label = 'Validation loss')
plt.title('Training and validation loss')
plt.legend()

plt.show()
Epoch 90/100
100/100 [==============================] - 601s 6s/step - loss: 0.0332 - acc: 0.9907 - val_loss: 0.2439 - val_acc: 0.9690
Epoch 91/100
100/100 [==============================] - 578s 6s/step - loss: 0.0043 - acc: 0.9974 - val_loss: 0.2387 - val_acc: 0.9690
Epoch 92/100
100/100 [==============================] - 597s 6s/step - loss: 0.0034 - acc: 0.9994 - val_loss: 0.2860 - val_acc: 0.9640
Epoch 93/100
100/100 [==============================] - 565s 6s/step - loss: 0.0019 - acc: 0.9993 - val_loss: 0.2611 - val_acc: 0.9650
Epoch 94/100
100/100 [==============================] - 608s 6s/step - loss: 0.0077 - acc: 0.9976 - val_loss: 0.2783 - val_acc: 0.9690
Epoch 95/100
100/100 [==============================] - 601s 6s/step - loss: 0.0072 - acc: 0.9970 - val_loss: 0.2546 - val_acc: 0.9660
Epoch 96/100
100/100 [==============================] - 583s 6s/step - loss: 0.0116 - acc: 0.9978 - val_loss: 0.2043 - val_acc: 0.9720
Epoch 97/100
100/100 [==============================] - 570s 6s/step - loss: 0.0122 - acc: 0.9969 - val_loss: 0.1967 - val_acc: 0.9610
Epoch 98/100
100/100 [==============================] - 560s 6s/step - loss: 0.0141 - acc: 0.9939 - val_loss: 0.5010 - val_acc: 0.9500
Epoch 99/100
100/100 [==============================] - 564s 6s/step - loss: 0.0341 - acc: 0.9945 - val_loss: 0.2402 - val_acc: 0.9640
Epoch 100/100
100/100 [==============================] - 572s 6s/step - loss: 7.3895e-04 - acc: 0.9996 - val_loss: 0.3101 - val_acc: 0.9710

可以看到,最终我们仅使用2000张图片就使得验证成功率在97%左右。