下载数据并划分数据‘

下载数据

这个只需注册一个kaggle账号，但这个比赛已经结束了，无法提交成绩

划分数据集

调用包

import numpy as np
import matplotlib.pyplot as plt
import os,shutil
import cv2
import glob as gb
import pandas as pd
import seaborn as sns
import matplotlib.image as mpimg
from tqdm import tqdm
from PIL import Image
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.callbacks import LearningRateScheduler,EarlyStopping,ReduceLROnPlateau
from tensorflow.keras import layers
from keras.preprocessing import image
from keras.preprocessing.image import ImageDataGenerator
from tensorflow.keras import optimizers
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report, confusion_matrix
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.applications import (DenseNet121,ResNet50V2,ResNet152V2,InceptionV3,Xception,ResNet50,VGG16,VGG19,MobileNet)

划分数据

original_dataset_dir = '换成你自己的'
base_dir = '写成你想要的'
os.mkdir(base_dir)
train_dir = os.path.join(base_dir, 'train')
os.mkdir(train_dir)
validation_dir = os.path.join(base_dir, 'validation')
os.mkdir(validation_dir)
test_dir = os.path.join(base_dir, 'test')
os.mkdir(test_dir)
train_cats_dir = os.path.join(train_dir, 'cats')
os.mkdir(train_cats_dir)
train_dogs_dir = os.path.join(train_dir, 'dogs')
os.mkdir(train_dogs_dir)
validation_cats_dir = os.path.join(validation_dir, 'cats')
os.mkdir(validation_cats_dir)
validation_dogs_dir = os.path.join(validation_dir, 'dogs')
os.mkdir(validation_dogs_dir)
test_cats_dir = os.path.join(test_dir, 'cats')
os.mkdir(test_cats_dir)
test_dogs_dir = os.path.join(test_dir, 'dogs')
os.mkdir(test_dogs_dir)
fnames = ['cat.{}.jpg'.format(i) for i in range(7000)]
for fname in fnames:
    src = os.path.join(original_dataset_dir, fname)
    dst = os.path.join(train_cats_dir, fname)
    shutil.copyfile(src, dst)
fnames = ['cat.{}.jpg'.format(i) for i in range(7000, 10000)]
for fname in fnames:
    src = os.path.join(original_dataset_dir, fname)
    dst = os.path.join(validation_cats_dir, fname)
    shutil.copyfile(src, dst)
fnames = ['cat.{}.jpg'.format(i) for i in range(10000, 12500)]
for fname in fnames:
    src = os.path.join(original_dataset_dir, fname)
    dst = os.path.join(test_cats_dir, fname)
    shutil.copyfile(src, dst)
fnames = ['dog.{}.jpg'.format(i) for i in range(7000)]
for fname in fnames:
    src = os.path.join(original_dataset_dir, fname)
    dst = os.path.join(train_dogs_dir, fname)
    shutil.copyfile(src, dst)
fnames = ['dog.{}.jpg'.format(i) for i in range(7000, 10000)]
for fname in fnames:
    src = os.path.join(original_dataset_dir, fname)
    dst = os.path.join(validation_dogs_dir, fname)
    shutil.copyfile(src, dst)
fnames = ['dog.{}.jpg'.format(i) for i in range(10000, 12500)]
for fname in fnames:
    src = os.path.join(original_dataset_dir, fname)
    dst = os.path.join(test_dogs_dir, fname)
    shutil.copyfile(src, dst)

original_dataset_dir就是你解压文件后的路径，base_dir是你想设置的路径，然后在base_dir路径下划分训练集，验证集和测试集，原始数据每个类别有12500张图像，我们使用7000张作为训练集，3000张作为验证集，2500张作为测试集。

数据处理，调参和建模

设置数据增强

datagen = ImageDataGenerator(rescale=1.0/255,rotation_range=30,width_shift_range=0.2,height_shift_range=0.2,shear_range=0.2,
                             zoom_range=0.2,horizontal_flip=True,fill_mode='nearest')
fnames = [os.path.join(train_cats_dir, fname) for fname in os.listdir(train_cats_dir)]
img_path = fnames[1200]
img = image.load_img(img_path, target_size=(150, 150))
x = image.img_to_array(img)
x = x.reshape((1,) + x.shape)
i = 0
for batch in datagen.flow(x, batch_size=1):
    plt.figure(i)
    imgplot = plt.imshow(image.array_to_img(batch[0]))
    i += 1
    if i % 4 == 0:
        break

对图像进行归一化，选择第1200张猫的图像展示数据增强效果

train_datagen = ImageDataGenerator(rescale=1.0/255,rotation_range=30,width_shift_range=0.2,height_shift_range=0.2,shear_range=0.2,
                                   zoom_range=0.2,horizontal_flip=True,fill_mode='nearest')
test_datagen = ImageDataGenerator(rescale=1./255)#验证数据不能使用数据增强
train_generator = train_datagen.flow_from_directory(train_dir,
                                                    target_size=(150, 150),batch_size=64,class_mode='binary',shuffle=True)
validation_generator = test_datagen.flow_from_directory(validation_dir,
                                                        target_size=(150, 150),batch_size=64,class_mode='binary',shuffle=True)

这里对于验证集和测试集不能使用数据增强，同时使用shuffle为True，对数据进行打乱，避免模型记住数据排列的顺序，否则模型会变得无效

选择模型的卷积基

#学习率指数衰减
learning_rate_schedule = keras.optimizers.schedules.ExponentialDecay(initial_learning_rate=0.01,
                                                                     decay_steps=1000,decay_rate=0.5,)
#比较不同卷积基
images_size=150
TL_Models =[
    MobileNet(input_shape=(images_size, images_size, 3), weights='imagenet', include_top=False),
    VGG19(input_shape=(images_size, images_size, 3), weights='imagenet', include_top=False),
    VGG16(input_shape=(images_size, images_size, 3), weights='imagenet', include_top=False),
    ResNet50(input_shape=(images_size, images_size, 3), weights='imagenet', include_top=False),
    ResNet50V2(input_shape=(images_size, images_size, 3), weights='imagenet', include_top=False),
    ResNet152V2(input_shape=(images_size, images_size, 3), weights='imagenet', include_top=False),
    InceptionV3(input_shape=(images_size, images_size, 3), weights='imagenet', include_top=False),
    Xception(input_shape=(images_size, images_size, 3), weights='imagenet', include_top=False),
    DenseNet121( input_shape=(images_size, images_size, 3) ,weights='imagenet', include_top=False),
]
TL_Models_NAMES = ['MobileNet','VGG19','VGG16','ResNet50','ResNet50V2',
                   'ResNet152V2','InceptionV3','Xception','DenseNet121']
for tl_model in TL_Models:
    tl_model.trainable = False
#训练模型
batch_size=64
from keras.models import Sequential
HISTORIES = []
for i in TL_Models:
    conv_base=i
    model = Sequential()
    model.add(conv_base)
    model.add(layers.Flatten())
    model.add(layers.Dense(256, activation='relu'))
    model.add(layers.Dense(1, activation='sigmoid'))
    model.compile(loss='binary_crossentropy',
                  optimizer=Adam(learning_rate=learning_rate_schedule),metrics='accuracy')
    history = model.fit(train_generator,steps_per_epoch=train_generator.samples // batch_size,epochs=10,
                              validation_data=validation_generator,
                              validation_steps=validation_generator.samples // batch_size)
    HISTORIES.append(history.history)
#绘图选择卷积基
styles=[':','-.','--','-',':','-.','--','-',':','-.','--','-']
plt.figure(figsize=(10, 8))
plt.subplot(2, 1, 1)
for i in range(len(HISTORIES)):
    plt.plot(HISTORIES[i]['val_accuracy'], linestyle=styles[i])
plt.legend(TL_Models_NAMES, loc='upper left')
plt.title('Validation Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.subplot(2, 1, 2)
for i in range(len(HISTORIES)):
    plt.plot(HISTORIES[i]['val_loss'], linestyle=styles[i])
plt.legend(TL_Models_NAMES, loc='upper left')
plt.title('Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')

分别选择不同的卷积基对模型进行训练最终结果如下：

最终我们选择模型的卷积基为ResNET152v2

进行模型微调

我们对卷积基的最后一个卷积块进行解冻，进行微调，学习率设置尽可能低，避免模型变化过大，底层的表示是通用的，不需要解冻

conv_base = ResNet152V2(input_shape=(images_size, images_size, 3), weights='imagenet', include_top=False)
conv_base.summary()
conv_base.trainable = True
set_trainable = False
for layer in conv_base.layers:
    if layer.name == 'conv5_block1_preact_bn ':
        set_trainable = True
    if set_trainable:
        layer.trainable = True
    else:
        layer.trainable = False

调整dropout参数

from keras.models import Sequential
nets = 8
model = [0] * nets
for j in range(8):
    model[j] = Sequential()
    model[j].add(conv_base)
    model[j].add(layers.Flatten())
    model[j].add(layers.Dense(256, activation='relu'))
    model[j].add(layers.Dropout(j * 0.1))
    model[j].add(layers.Dense(1, activation='sigmoid'))
    model[j].compile(loss='binary_crossentropy',
                  optimizer=Adam(learning_rate=learning_rate_schedule), metrics='accuracy')
history = [0] * nets
names = ["0", "1", "2", "3", "4", "5",'6', '7']
epochs=10
for j in range(nets):
    history[j] = model[j].fit(train_generator, steps_per_epoch=train_generator.samples // batch_size, epochs=10,
                              validation_data=validation_generator,
                              validation_steps=validation_generator.samples // batch_size)
    print("CNN {0}: Epochs={1:d}, Train accuracy={2:.5f}, Validation accuracy={3:.5f}".format(
        names[j], epochs, max(history[j].history['accuracy']), max(history[j].history['val_accuracy'])))
    plt.plot(history[j].history['val_accuracy'], linestyle=styles[j])
    plt.legend(names, loc='upper left')

过程大概下面这样，搞了一晚上

看下面结果，不用dropout

训练并保存最优模型

early_stopping = EarlyStopping(min_delta=1e-7, patience=4, restore_best_weights=True,)
learning_rate_reduce = ReduceLROnPlateau(monitor='val_acc',patience=4,verbose=1,factor=0.5,min_lr=1e-8)
modelCheckpoint=keras.callbacks.ModelCheckpoint(filepath='my_best__model.h5',monitor='val_acc',save_best_only=True,)
callbacks_list = [early_stopping,learning_rate_reduce,modelCheckpoint]
model = Sequential()
model.add(conv_base)
model.add(layers.Flatten())
model.add(layers.Dense(256, activation='relu'))
model.add(layers.BatchNormalization())
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',optimizer=optimizers.RMSprop(learning_rate=2e-5),metrics=['acc'])
history = model.fit(train_generator,steps_per_epoch=train_generator.samples // batch_size,epochs=150,
                              validation_data=validation_generator,
                              validation_steps=validation_generator.samples // batch_size,
                    callbacks=[callbacks_list])
from keras.models import load_model
model = load_model('my_best__model.h5')

训练并保存最佳模型，然后就用在测试集上，

用保留的最优模型进行预测

test_generator = test_datagen.flow_from_directory(test_dir,
                                                  target_size=(150, 150),batch_size=64,class_mode='binary',shuffle=True)
score = model.evaluate(test_generator, verbose=False)
print('Test loss:', score[0])
print('Test accuracy:', score[1])

97%正确率

可视化模型

from tensorflow.keras.utils import plot_model
plot_model(model, show_shapes=True, to_file='model.png')