项目11:基于深度学习的面部情绪识别算法

1. 准备工作

在指定的磁盘路径创建存放当前项目的目录,linux或macos可使用mkdir命令创建文件夹目录,Windows直接使用图形化界面右键新建文件夹即可,例如我们的存放项目的目录名为project11:

    (dlwork) jingyudeMacBook-Pro:~ jingyuyan$ mkdir project11

进入project10文件夹后,启动jupyter,创建一个文件开始我们的实验。

    (dlwork) jingyudeMacBook-Pro:~ jingyuyan$ cd project11

    (dlwork) jingyudeMacBook-Pro:project11$ jupyter notebook

2. Fer2013人脸表情数处理

请读者翻页附录获取下载fer2013.csv数据集和test_set文件夹的方法。Fer2013人脸表情数据集由35886张人脸表情图片组成,其中,训练集28708张,测试集和私有验证集各3589张图片,每张图片是由大小固定为48×48的灰度图像组成,共有7种表情,分别对应于数字标签0-6,具体表情对应的标签和中英文如下:

  • 0 anger 生气
  • 1 disgust 厌恶 
  • 2 fear 恐惧
  • 3 happy 开心
  • 4 sad 伤心
  • 5 surprised 惊讶
  • 6 normal 自然。

在根目录下创建build目录,将fer2013.csv文件放入build中、test_set文件夹放入项目根目录下;并在根目录创建datasets文件夹,同时将上一章节所提及到的models文件夹和mtcnnplug文件夹拷贝到根目录下(仅使用到mtcnn功能,读者可以自行剔除haarcascades相关的部分文件或者原封不动复制也不影响项目运行)。读者可以得到以下目录:

project11/
├───demo11.ipynb
├───build/
├───datasets/
├───model/
  └───mtcnn.mlz
├───mtcnnplug/
      ├──__init__.py
      ├──native.py
      └──core/
         ├──libface_python_ext.dll
         └──libface_python_ext.so
└───test_set
        ├──emojis/
         ├──angry.png
         ├──disgusted.png
         ├──fearful.png
          .....
        └──test_imgs/
           ├──01.jpeg
           └──02.jpg

2.1 数据集拆解与划分

我们先读取并构建各个数据集存放的目录,并对数据集文件进行拆解和划分。

# 导包
import matplotlib.pyplot as plt
import csv
import os
import csv
import os
from PIL import Image
import numpy as np
import cv2
# 将CSV文件导划分训练集、测试集和验证集三个集合

database_path = 'build/'
datasets_path = 'datasets/'
csv_file = database_path + 'fer2013.csv'
train_csv = datasets_path + 'train.csv'
val_csv = datasets_path + 'val.csv'
test_csv = datasets_path + 'test.csv'

with open(csv_file) as f:
    csvr = csv.reader(f)
    header = next(csvr)
    print(header)
    rows = [row for row in csvr]

    trn = [row[:-1] for row in rows if row[-1] == 'Training']
    csv.writer(open(train_csv, 'w+'), lineterminator='\n').writerows([header[:-1]] + trn)
    print('Training :', len(trn))

    val = [row[:-1] for row in rows if row[-1] == 'PublicTest']
    csv.writer(open(val_csv, 'w+'), lineterminator='\n').writerows([header[:-1]] + val)
    print('PublicTest : ', len(val))

    tst = [row[:-1] for row in rows if row[-1] == 'PrivateTest']
    csv.writer(open(test_csv, 'w+'), lineterminator='\n').writerows([header[:-1]] + tst)
    print('PrivateTest', len(tst))
['emotion', 'pixels', 'Usage']
Training : 28709
PublicTest :  3589
PrivateTest 3589

2.2 将数据转换为图片和标签形式

成功划分和转成成图片后的数据需要将其全部转换成单通道的灰度图片。

# 转换图片和标签
datasets_path = 'datasets/'
train_csv = os.path.join(datasets_path, 'train.csv')
val_csv = os.path.join(datasets_path, 'val.csv')
test_csv = os.path.join(datasets_path, 'test.csv')

train_set = os.path.join(datasets_path, 'train')
val_set = os.path.join(datasets_path, 'val')
test_set = os.path.join(datasets_path, 'test')

for save_path, csv_file in [(train_set, train_csv), (val_set, val_csv), (test_set, test_csv)]:
    if not os.path.exists(save_path):
        os.makedirs(save_path)

    num = 1
    with open(csv_file) as f:
        csvr = csv.reader(f)
        header = next(csvr)
        for i, (label, pixel) in enumerate(csvr):
            pixel = np.asarray([float(p) for p in pixel.split()]).reshape(48, 48)
            subfolder = os.path.join(save_path, label)
            if not os.path.exists(subfolder):
                os.makedirs(subfolder)
            im = Image.fromarray(pixel).convert('L')
            image_name = os.path.join(subfolder, '{:05d}.jpg'.format(i))
            im.save(image_name)

编写可视化函数,将转换后的数据绘制出来,方便查看

from matplotlib.font_manager import FontProperties
font_zh = FontProperties(fname='./fz.ttf')
EMOTION_LABELS = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral']
EMOTION_LABELS_CH = ['生气', '厌恶', '恐惧', '开心', '伤心', '惊讶', '自然']
def plot_image_labels_prediction(images,labels,num=35):
    fig = plt.gcf()
    fig.set_size_inches(12,14)
    for i in range(0,num):
        ax = plt.subplot(7,5,1+i)
        ax.imshow(images[i],cmap='binary')
        title = str(i+1) + ' ' +EMOTION_LABELS_CH[labels[i]]
        ax.set_title(title,fontproperties = font_zh, fontsize=10)
        ax.set_xticks([]);ax.set_yticks([])
    plt.show()
# 选取训练集中的前五张图片进行查看
images = []
labels = []
start = 50
count = 5
for typ in range(0, 7):
    for file in os.listdir(os.path.join(train_set, str(typ)))[start : start + count]:
        images.append(cv2.imread(os.path.join(train_set, str(typ), file)))
        labels.append(typ)

# 绘制图像
plot_image_labels_prediction(images, labels)
images[0].shape

png

(48, 48, 3)

3. 情绪分类器训练

3.1 搭建模型

设置训练参数,并采用VGG16网络进行改进,组成一个更加深度的卷积神经网络。

# 导包
import keras
from keras.layers import Dense, Dropout, Activation, Flatten,Conv2D, MaxPooling2D
from keras.layers.normalization import BatchNormalization
from keras.models import Sequential
from keras.preprocessing.image import ImageDataGenerator
from keras.callbacks import EarlyStopping
from keras.optimizers import SGD
Using TensorFlow backend.

设置训练所用到的参数

# 每次喂入训练数据量
batch_siz = 128
# 分类数量为7(7种表情)
num_classes = 7
# 批次
nb_epoch = 70
# 图像尺寸
img_size=48
# 数据集根目录
root_path='datasets/'
# 模型存储目录
models_path='models/'
class Model:
    """
        build_model - 构建模型
        train_model - 训练模型
        save_model  - 存储模型权重
    """
    def __init__(self):
        self.model = None

    def build_model(self):
        self.model = Sequential()

        self.model.add(Conv2D(32, (1, 1), strides=1, padding='same', input_shape=(img_size, img_size, 1)))
        self.model.add(Activation('relu'))
        self.model.add(Conv2D(32, (5, 5), padding='same'))
        self.model.add(Activation('relu'))
        self.model.add(MaxPooling2D(pool_size=(2, 2)))

        self.model.add(Conv2D(32, (3, 3), padding='same'))
        self.model.add(Activation('relu'))
        self.model.add(MaxPooling2D(pool_size=(2, 2)))

        self.model.add(Conv2D(64, (5, 5), padding='same'))
        self.model.add(Activation('relu'))
        self.model.add(MaxPooling2D(pool_size=(2, 2)))

        self.model.add(Flatten())
        self.model.add(Dense(2048))
        self.model.add(Activation('relu'))
        self.model.add(Dropout(0.5))
        self.model.add(Dense(1024))
        self.model.add(Activation('relu'))
        self.model.add(Dropout(0.5))
        self.model.add(Dense(num_classes))
        self.model.add(Activation('softmax'))
        self.model.summary()


    def train_model(self):
        sgd=SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
        self.model.compile(loss='categorical_crossentropy',
                optimizer=sgd,
                #optimizer='rmsprop',
                metrics=['accuracy'])
        # 样本的扩充设置
        train_datagen = ImageDataGenerator(
        rescale = 1./255,
        shear_range = 0.2,
        zoom_range = 0.2,
        horizontal_flip=True)
        # 验证
        val_datagen = ImageDataGenerator(
                rescale = 1./255)
        eval_datagen = ImageDataGenerator(
                rescale = 1./255)
        # 保存扩充样本
        train_generator = train_datagen.flow_from_directory(
                root_path+'/train',
                target_size=(img_size, img_size),
                color_mode='grayscale',
                batch_size=batch_siz,
                class_mode='categorical')
        val_generator = val_datagen.flow_from_directory(
                root_path+'/val',
                target_size=(img_size, img_size),
                color_mode='grayscale',
                batch_size=batch_siz,
                class_mode='categorical')
        eval_generator = eval_datagen.flow_from_directory(
                root_path+'/test',
                target_size=(img_size, img_size),
                color_mode='grayscale',
                batch_size=batch_siz,
                class_mode='categorical')
        early_stopping = EarlyStopping(monitor='loss',patience=3)
        history_fit=self.model.fit_generator(
                train_generator,
                steps_per_epoch=800/(batch_siz/32),#28709
                nb_epoch=nb_epoch,
                validation_data=val_generator,
                validation_steps=2000,
                )
        history_predict=self.model.predict_generator(
                eval_generator,
                steps=2000)
        with open(root_path+'/model_fit_log','w') as f:
            f.write(str(history_fit.history))
        with open(root_path+'/model_predict_log','w') as f:
            f.write(str(history_predict))
        print('model trained')

    # 模型存储设置
    def save_model(self):
        model_json=self.model.to_json()
        with open(models_path+"/model_json.json", "w") as json_file:
            json_file.write(model_json)
        self.model.save_weights(models_path+'/model_weight.h5')
        self.model.save(models_path+'/model.h5')
        print('model saved')
# 创建模型 输出模型摘要
model=Model()
model.build_model()
WARNING: Logging before flag parsing goes to stderr.
W0115 03:02:37.084475 4579337664 deprecation_wrapper.py:119] From /Users/jingyuyan/anaconda3/envs/dlwork/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py:4070: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.



Model: "sequential_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
conv2d_1 (Conv2D)            (None, 48, 48, 32)        64        
_________________________________________________________________
activation_1 (Activation)    (None, 48, 48, 32)        0         
_________________________________________________________________
conv2d_2 (Conv2D)            (None, 48, 48, 32)        25632     
_________________________________________________________________
activation_2 (Activation)    (None, 48, 48, 32)        0         
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 24, 24, 32)        0         
_________________________________________________________________
conv2d_3 (Conv2D)            (None, 24, 24, 32)        9248      
_________________________________________________________________
activation_3 (Activation)    (None, 24, 24, 32)        0         
_________________________________________________________________
max_pooling2d_2 (MaxPooling2 (None, 12, 12, 32)        0         
_________________________________________________________________
conv2d_4 (Conv2D)            (None, 12, 12, 64)        51264     
_________________________________________________________________
activation_4 (Activation)    (None, 12, 12, 64)        0         
_________________________________________________________________
max_pooling2d_3 (MaxPooling2 (None, 6, 6, 64)          0         
_________________________________________________________________
flatten_1 (Flatten)          (None, 2304)              0         
_________________________________________________________________
dense_1 (Dense)              (None, 2048)              4720640   
_________________________________________________________________
activation_5 (Activation)    (None, 2048)              0         
_________________________________________________________________
dropout_1 (Dropout)          (None, 2048)              0         
_________________________________________________________________
dense_2 (Dense)              (None, 1024)              2098176   
_________________________________________________________________
activation_6 (Activation)    (None, 1024)              0         
_________________________________________________________________
dropout_2 (Dropout)          (None, 1024)              0         
_________________________________________________________________
dense_3 (Dense)              (None, 7)                 7175      
_________________________________________________________________
activation_7 (Activation)    (None, 7)                 0         
=================================================================
Total params: 6,912,199
Trainable params: 6,912,199
Non-trainable params: 0
_________________________________________________________________

如图所示模型结构

开始训练模型,训练时间较长,推荐使用GPU

# 开始训练 并且保存模型权重
model.train_model()
model.save_model()
Found 28709 images belonging to 7 classes.
Found 3589 images belonging to 7 classes.
Found 3589 images belonging to 7 classes.


/home/tunm/anaconda3/envs/dlwork/lib/python3.6/site-packages/ipykernel_launcher.py:81: UserWarning: The semantics of the Keras 2 argument `steps_per_epoch` is not the same as the Keras 1 argument `samples_per_epoch`. `steps_per_epoch` is the number of batches to draw from the generator at each epoch. Basically steps_per_epoch = samples_per_epoch/batch_size. Similarly `nb_val_samples`->`validation_steps` and `val_samples`->`steps` arguments have changed. Update your method calls accordingly.
/home/tunm/anaconda3/envs/dlwork/lib/python3.6/site-packages/ipykernel_launcher.py:81: UserWarning: Update your `fit_generator` call to the Keras 2 API: `fit_generator(<keras_pre..., steps_per_epoch=200.0, validation_data=<keras_pre..., validation_steps=2000, epochs=70)`


Epoch 1/70
200/200 [==============================] - 79s 396ms/step - loss: 1.8232 - acc: 0.2451 - val_loss: 1.8019 - val_acc: 0.2494
Epoch 2/70
200/200 [==============================] - 70s 350ms/step - loss: 1.7900 - acc: 0.2523 - val_loss: 1.7628 - val_acc: 0.2750
Epoch 3/70
200/200 [==============================] - 70s 351ms/step - loss: 1.7575 - acc: 0.2727 - val_loss: 1.7161 - val_acc: 0.3182
Epoch 4/70
200/200 [==============================] - 69s 347ms/step - loss: 1.6968 - acc: 0.3249 - val_loss: 1.6196 - val_acc: 0.3739
Epoch 5/70
200/200 [==============================] - 70s 351ms/step - loss: 1.6253 - acc: 0.3624 - val_loss: 1.5452 - val_acc: 0.4060
Epoch 6/70
200/200 [==============================] - 70s 352ms/step - loss: 1.5802 - acc: 0.3882 - val_loss: 1.4890 - val_acc: 0.4361
Epoch 7/70
200/200 [==============================] - 71s 353ms/step - loss: 1.5383 - acc: 0.4030 - val_loss: 1.4485 - val_acc: 0.4461
Epoch 8/70
200/200 [==============================] - 68s 340ms/step - loss: 1.4975 - acc: 0.4214 - val_loss: 1.4151 - val_acc: 0.4611
Epoch 9/70
200/200 [==============================] - 70s 348ms/step - loss: 1.4678 - acc: 0.4369 - val_loss: 1.3882 - val_acc: 0.4753
Epoch 10/70
200/200 [==============================] - 70s 350ms/step - loss: 1.4282 - acc: 0.4496 - val_loss: 1.3446 - val_acc: 0.4882
Epoch 11/70
200/200 [==============================] - 71s 354ms/step - loss: 1.4009 - acc: 0.4620 - val_loss: 1.3157 - val_acc: 0.4943
Epoch 12/70
200/200 [==============================] - 71s 354ms/step - loss: 1.3684 - acc: 0.4757 - val_loss: 1.2926 - val_acc: 0.5007
Epoch 13/70
200/200 [==============================] - 70s 350ms/step - loss: 1.3521 - acc: 0.4821 - val_loss: 1.2882 - val_acc: 0.5057
Epoch 14/70
200/200 [==============================] - 70s 350ms/step - loss: 1.3125 - acc: 0.4961 - val_loss: 1.2475 - val_acc: 0.5208
Epoch 15/70
200/200 [==============================] - 71s 356ms/step - loss: 1.2926 - acc: 0.5033 - val_loss: 1.2357 - val_acc: 0.5210
Epoch 16/70
200/200 [==============================] - 71s 355ms/step - loss: 1.2775 - acc: 0.5164 - val_loss: 1.2259 - val_acc: 0.5286
Epoch 17/70
200/200 [==============================] - 70s 351ms/step - loss: 1.2523 - acc: 0.5237 - val_loss: 1.2309 - val_acc: 0.5244
Epoch 18/70
200/200 [==============================] - 71s 355ms/step - loss: 1.2417 - acc: 0.5294 - val_loss: 1.2136 - val_acc: 0.5358
Epoch 19/70
200/200 [==============================] - 70s 349ms/step - loss: 1.2275 - acc: 0.5367 - val_loss: 1.2035 - val_acc: 0.5417
Epoch 20/70
200/200 [==============================] - 71s 355ms/step - loss: 1.2027 - acc: 0.5422 - val_loss: 1.1849 - val_acc: 0.5442
Epoch 21/70
200/200 [==============================] - 71s 356ms/step - loss: 1.1892 - acc: 0.5482 - val_loss: 1.1724 - val_acc: 0.5539
Epoch 22/70
200/200 [==============================] - 71s 353ms/step - loss: 1.1791 - acc: 0.5551 - val_loss: 1.1697 - val_acc: 0.5556
Epoch 23/70
200/200 [==============================] - 70s 349ms/step - loss: 1.1614 - acc: 0.5602 - val_loss: 1.1688 - val_acc: 0.5564
Epoch 24/70
200/200 [==============================] - 71s 355ms/step - loss: 1.1516 - acc: 0.5644 - val_loss: 1.1506 - val_acc: 0.5612
Epoch 25/70
200/200 [==============================] - 71s 355ms/step - loss: 1.1280 - acc: 0.5724 - val_loss: 1.1688 - val_acc: 0.5525
Epoch 26/70
200/200 [==============================] - 70s 352ms/step - loss: 1.1342 - acc: 0.5708 - val_loss: 1.1471 - val_acc: 0.5692
Epoch 27/70
200/200 [==============================] - 71s 353ms/step - loss: 1.1140 - acc: 0.5772 - val_loss: 1.1439 - val_acc: 0.5706
Epoch 28/70
200/200 [==============================] - 70s 350ms/step - loss: 1.1071 - acc: 0.5827 - val_loss: 1.1368 - val_acc: 0.5692
Epoch 29/70
200/200 [==============================] - 70s 352ms/step - loss: 1.0913 - acc: 0.5879 - val_loss: 1.1243 - val_acc: 0.5717
Epoch 30/70
200/200 [==============================] - 70s 351ms/step - loss: 1.0814 - acc: 0.5963 - val_loss: 1.1476 - val_acc: 0.5729
Epoch 31/70
200/200 [==============================] - 71s 354ms/step - loss: 1.0770 - acc: 0.5966 - val_loss: 1.1225 - val_acc: 0.5765
Epoch 32/70
200/200 [==============================] - 70s 350ms/step - loss: 1.0707 - acc: 0.5942 - val_loss: 1.1239 - val_acc: 0.5765
Epoch 33/70
200/200 [==============================] - 70s 348ms/step - loss: 1.0470 - acc: 0.6054 - val_loss: 1.1173 - val_acc: 0.5876
Epoch 34/70
200/200 [==============================] - 70s 351ms/step - loss: 1.0413 - acc: 0.6050 - val_loss: 1.1194 - val_acc: 0.5851
Epoch 35/70
200/200 [==============================] - 70s 352ms/step - loss: 1.0265 - acc: 0.6107 - val_loss: 1.1071 - val_acc: 0.5868
Epoch 36/70
200/200 [==============================] - 70s 348ms/step - loss: 1.0282 - acc: 0.6123 - val_loss: 1.1145 - val_acc: 0.5804
Epoch 37/70
200/200 [==============================] - 70s 349ms/step - loss: 1.0016 - acc: 0.6240 - val_loss: 1.1077 - val_acc: 0.5929
Epoch 38/70
200/200 [==============================] - 69s 346ms/step - loss: 1.0028 - acc: 0.6217 - val_loss: 1.1062 - val_acc: 0.5832
Epoch 39/70
200/200 [==============================] - 70s 350ms/step - loss: 0.9897 - acc: 0.6308 - val_loss: 1.1153 - val_acc: 0.5871
Epoch 40/70
200/200 [==============================] - 70s 352ms/step - loss: 0.9719 - acc: 0.6340 - val_loss: 1.1044 - val_acc: 0.5929
Epoch 41/70
200/200 [==============================] - 71s 356ms/step - loss: 0.9783 - acc: 0.6360 - val_loss: 1.0968 - val_acc: 0.5879
Epoch 42/70
200/200 [==============================] - 70s 348ms/step - loss: 0.9589 - acc: 0.6391 - val_loss: 1.1174 - val_acc: 0.5968
Epoch 43/70
200/200 [==============================] - 70s 352ms/step - loss: 0.9609 - acc: 0.6407 - val_loss: 1.1013 - val_acc: 0.6004
Epoch 44/70
200/200 [==============================] - 70s 348ms/step - loss: 0.9396 - acc: 0.6458 - val_loss: 1.1093 - val_acc: 0.5996
Epoch 45/70
200/200 [==============================] - 70s 351ms/step - loss: 0.9353 - acc: 0.6509 - val_loss: 1.1018 - val_acc: 0.5993
Epoch 46/70
200/200 [==============================] - 70s 351ms/step - loss: 0.9186 - acc: 0.6561 - val_loss: 1.1103 - val_acc: 0.6016
Epoch 47/70
200/200 [==============================] - 70s 352ms/step - loss: 0.9086 - acc: 0.6610 - val_loss: 1.1091 - val_acc: 0.6013
Epoch 48/70
200/200 [==============================] - 72s 361ms/step - loss: 0.9092 - acc: 0.6564 - val_loss: 1.1003 - val_acc: 0.6016
Epoch 49/70
200/200 [==============================] - 72s 359ms/step - loss: 0.8941 - acc: 0.6635 - val_loss: 1.0905 - val_acc: 0.6080
Epoch 50/70
200/200 [==============================] - 72s 358ms/step - loss: 0.8814 - acc: 0.6666 - val_loss: 1.0948 - val_acc: 0.6082
Epoch 51/70
200/200 [==============================] - 71s 356ms/step - loss: 0.8734 - acc: 0.6745 - val_loss: 1.1103 - val_acc: 0.6069
Epoch 52/70
200/200 [==============================] - 72s 360ms/step - loss: 0.8652 - acc: 0.6785 - val_loss: 1.1243 - val_acc: 0.6041
Epoch 53/70
200/200 [==============================] - 71s 353ms/step - loss: 0.8498 - acc: 0.6855 - val_loss: 1.1311 - val_acc: 0.6018
Epoch 54/70
200/200 [==============================] - 71s 357ms/step - loss: 0.8486 - acc: 0.6845 - val_loss: 1.1112 - val_acc: 0.6029
Epoch 55/70
200/200 [==============================] - 72s 359ms/step - loss: 0.8337 - acc: 0.6915 - val_loss: 1.1181 - val_acc: 0.6066
Epoch 56/70
200/200 [==============================] - 72s 360ms/step - loss: 0.8293 - acc: 0.6887 - val_loss: 1.1027 - val_acc: 0.6133
Epoch 57/70
200/200 [==============================] - 72s 358ms/step - loss: 0.8061 - acc: 0.6998 - val_loss: 1.1266 - val_acc: 0.6105
Epoch 58/70
200/200 [==============================] - 71s 356ms/step - loss: 0.8055 - acc: 0.6996 - val_loss: 1.1039 - val_acc: 0.6122
Epoch 59/70
200/200 [==============================] - 71s 355ms/step - loss: 0.7905 - acc: 0.7054 - val_loss: 1.1789 - val_acc: 0.6113
Epoch 60/70
200/200 [==============================] - 71s 353ms/step - loss: 0.7857 - acc: 0.7065 - val_loss: 1.1624 - val_acc: 0.6088
Epoch 61/70
200/200 [==============================] - 71s 355ms/step - loss: 0.7689 - acc: 0.7139 - val_loss: 1.1487 - val_acc: 0.6155
Epoch 62/70
200/200 [==============================] - 71s 353ms/step - loss: 0.7619 - acc: 0.7190 - val_loss: 1.1517 - val_acc: 0.6147
Epoch 63/70
200/200 [==============================] - 70s 351ms/step - loss: 0.7563 - acc: 0.7186 - val_loss: 1.1148 - val_acc: 0.6183
Epoch 64/70
200/200 [==============================] - 72s 358ms/step - loss: 0.7488 - acc: 0.7222 - val_loss: 1.1319 - val_acc: 0.6155
Epoch 65/70
200/200 [==============================] - 71s 355ms/step - loss: 0.7289 - acc: 0.7293 - val_loss: 1.1311 - val_acc: 0.6191
Epoch 66/70
200/200 [==============================] - 70s 352ms/step - loss: 0.7311 - acc: 0.7309 - val_loss: 1.1403 - val_acc: 0.6144
Epoch 67/70
200/200 [==============================] - 71s 353ms/step - loss: 0.7281 - acc: 0.7289 - val_loss: 1.1417 - val_acc: 0.6236
Epoch 68/70
200/200 [==============================] - 70s 352ms/step - loss: 0.7171 - acc: 0.7345 - val_loss: 1.1564 - val_acc: 0.6188
Epoch 69/70
200/200 [==============================] - 71s 353ms/step - loss: 0.6922 - acc: 0.7450 - val_loss: 1.1663 - val_acc: 0.6202
Epoch 70/70
200/200 [==============================] - 70s 351ms/step - loss: 0.6960 - acc: 0.7432 - val_loss: 1.1572 - val_acc: 0.6183
model trained
model saved

3. 使用MTCNN人脸检测模块

上一章具体阐述过MTCNN人脸检测的使用,请读者使用作者训练好的MTCNN模型进行实验。首先,构建MTCNN的各个组建路径。

# 获取项目根目录
ROOT = os.getcwd()
# 构建models文件夹目录
MODELS_PATH = os.path.join(ROOT, 'models')
# 构建mtcnnplug目录
MTCNN_PLUG_PATH = os.path.join(ROOT, 'mtcnnplug')
# 构建mtcnn中的core目录
MTCNN_CORE_PATH = os.path.join(MTCNN_PLUG_PATH, 'core')
# 构建mtcnn中的model路径
MTCNN_MODEL = os.path.join(MODELS_PATH, 'mtcnn.mlz')
# 构建core目录下的动态链接库路径,根据系统选择不一样的动态链接库
MTCNN_LIBFACE = os.path.join(MTCNN_CORE_PATH, 'libface_python_ext.so')
# MTCNN_LIBFACE = os.path.join(MTCNN_CORE_PATH, 'libface_python_ext.dll')

构建mtcnn人脸检测器对象

from mtcnnplug.native import MtcnnFaceDetector
mtcnnFaceDetector = MtcnnFaceDetector(MTCNN_LIBFACE, MTCNN_MODEL)

构建faces_detection_mtcnn和单图显示函数imshow检测函数。

import time

def faces_detection_mtcnn(mtcnnFaceDetector, target, minSize=10, bdsize=2):
    # 记录当前时间
    current_time = time.time()
    # 检测test_img_gray对象
    bboxes = mtcnnFaceDetector.simpleDetection(target, minSizeImage=10)
    # 计算检测用时
    use_time = time.time() - current_time
    # 复制一张原图用于绘制
    target_draw = target.copy()
    if (len(bboxes) != 0):
        print("检测到的人脸数量: ", len(bboxes))
        print("检测结果的形状: ",bboxes.shape)
        print("检测用时: ", str(round(use_time, 5))+'秒')

        # 绘制
        for bbox in bboxes:
            x1, y1, x2, y2 = bbox[1:5]
            # 绘制人脸矩形框
            cv2.rectangle(target_draw, (x1, y1), (x2, y2), (200, 200,0), bdsize)
            # 绘制人脸五个关键点
            for i in range(5, 15, 2):
                cv2.circle(target_draw, (int(bbox[i + 0]), int(bbox[i + 1])), bdsize, (200, 200,0), -1)
    else:
        print("未检测到人脸")

    # 这边添加一个输出属性bboxes
    return target_draw, bboxes

def imshow(img, size=(7, 7)):
    plt.figure(figsize=size)
    if len(img.shape) == 3:
        img2 = img[:,:,::-1] 
        plt.imshow(img2)
    elif len(img.shape) == 2:
        img2 = img[:,:]
        plt.imshow(img2, cmap='gray')
    else:
        print('error')

随机测试一个图像

# 读取和显示图片
test_img = cv2.imread('./test_set/test_imgs/01.jpeg')
res_test, _ = faces_detection_mtcnn(mtcnnFaceDetector, test_img)
imshow(res_test, size=(9, 9))
检测到的人脸数量:  2
检测结果的形状:  (2, 15)
检测用时:  0.17764

png

3. 预测模型

将训练结果进行预测,查看效果

3.1 构建表情检测器

# 导包
import numpy as np
import cv2
import sys
import json
import os
from keras.models import model_from_json
import time
from PIL import Image, ImageDraw, ImageFont 

配置一些模型需要用到的基本属性,我们在emoji表情中找了7个与我们表情预测结果的7个分类比较符合的表情来进行匹配。

# 基本配置
emojie_path = 'test_set/emojis'
md_json_path = 'models/model_json.json'
md_weight_path = 'models/model_weight.h5'
EMOTION_LABELS = ['angry', 'disgust', 'fear', 'happy', 'sad', 'surprise', 'neutral']
EMOTION_LABELS_CH = ['生气', '厌恶:', '害怕', '高兴', '悲伤', '惊讶', '自然']
emojis = ['angry', 'disgusted', 'fearful', 'happy', 'sad', 'surprised', 'neutral']
emoji_size = 20
text_color = (255, 255, 0)
font_path = 'fz.ttf'
IMG_SIZE = 48
NUM_CLASS = len(EMOTION_LABELS)

获取emoji表情图片文件

# 获取emoji表情图片
def getPathEmoji(index):
    emoji = emojis[index]
    efile = os.path.join(emojie_path, emoji) + '.png'
    if os.path.exists(efile):
        return(efile)
    else:
        return('error path')

构建EmotionDetector表情检测器

class EmotionDetector:
    """ 表情检测器 """

    def __init__(self, md_json_path, md_weight):
        """ 加载模型 """
        json_file = open(md_json_path)
        loaded_model_json = json_file.read()
        json_file.close()
        self.model = model_from_json(loaded_model_json)
        self.model.load_weights(md_weight)

    def predict_emotion(self, face_img):
        """ 预测结果 """
        face_img = face_img * (1. / 255)
        resized_img = cv2.resize(face_img, (IMG_SIZE, IMG_SIZE))
        rsz_img = []
        rsh_img = []
        results = []
        rsz_img.append(resized_img[:, :])
        rsz_img.append(resized_img[2:45, :])
        rsz_img.append(cv2.flip(rsz_img[0], 1))
        i = 0
        for rsz_image in rsz_img:
            rsz_img[i] = cv2.resize(rsz_image, (IMG_SIZE, IMG_SIZE))
            i += 1
        for rsz_image in rsz_img:
            rsh_img.append(rsz_image.reshape(1, IMG_SIZE, IMG_SIZE, 1))
        i = 0
        for rsh_image in rsh_img:
            list_of_list = self.model.predict_proba(
                rsh_image, batch_size=32, verbose=0)
            result = [prob for lst in list_of_list for prob in lst]
            results.append(result)
        return results

    def detection(self, img):
        """ 预测结果 + 格式化结果数据 """
        results = self.predict_emotion(img)
        result_sum = np.array([0] * NUM_CLASS)
        for result in results:
            result_sum = result_sum + np.array(result)
        angry, disgust, fear, happy, sad, surprise, neutral = result_sum
        label = np.argmax(result_sum)
        emo = EMOTION_LABELS[label]
        emo_ch = EMOTION_LABELS_CH[label]
        result_dict = {"label": label, "emotion": emo, "emotion_ch": emo_ch}
        return result_dict
# 建立绘制函数

def drawImgaeOfCH(image, text ,position, font_path, fontsize, fillColor):
    """ 图片绘制中文函数 """
    img_PIL = Image.fromarray(cv2.cvtColor(image, cv2.COLOR_BGR2RGB)) 
    font = ImageFont.truetype(font_path, fontsize)
    draw = ImageDraw.Draw(img_PIL) 
    draw.text(position, text, font=font, fill=fillColor)
    img_OpenCV = cv2.cvtColor(np.asarray(img_PIL),cv2.COLOR_RGB2BGR) 
    return img_OpenCV

def drawing_image_emotion(image, faces, emotionDetector, isCh=False, isDrawEmoji=False):
    """ 绘制出检测的结果 """
    draw = image.copy()
    for face in faces:
        x1, y1, x2, y2 = face[1:5]
        # 裁剪出检测到的人脸图片
        crop = image[int(y1): int(y2),int(x1): int(x2)]
        crop = cv2.cvtColor(crop, cv2.COLOR_BGR2GRAY)
        # 表情检测
        emotion_res = emotionDetector.detection(crop)
        # 绘制矩形框
        cv2.rectangle(draw, (int(x1), int(y1)), (int(x2), int(y2)),text_color, 4)
        if isCh:
            """ 是否显示中文结果 """
            draw = drawImgaeOfCH(draw, emotion_res['emotion_ch'], (int(x1), int(y1-30)), font_path, 30, text_color)
        else:
            cv2.putText(draw, emotion_res['emotion'], (int(x1), int(y2-3)), cv2.FONT_HERSHEY_SIMPLEX, 0.6,text_color, 2)
        if isDrawEmoji:
            """ 是否显示emoji表情 """
            emoji_path = getPathEmoji(int(emotion_res['label']))
            emoji = cv2.imread(emoji_path)
            emoji = cv2.resize(emoji, (emoji_size, emoji_size))
            draw[int(y1):int(y1+emoji_size), int(x1):int(x1+emoji_size)] = emoji[:, :]

    return draw


def time_it(method):                                                           

    def timed(*args, **kw):                                                    
        start_time = time.time()                                               
        result = method(*args, **kw)                                        
        end_time = time.time()                                                 
        print ('%r (%r, %r) %2.2f sec' % (method.__name__, args, kw, end_time-start_time))          
        return result                                                          

    return timed 

3.2 测试模型

我们使用EmotionDetector表情检测器和MtcnnFaceDetector人脸检测器搭配使用检测出图像中人脸的表情。

# 绘制出预测结果
# 读取和显示图片
test_img = cv2.imread('./test_set/test_imgs/01.jpeg')
# 先检测出人脸所在的位置
boxes = mtcnnFaceDetector.simpleDetection(test_img, minSizeImage=15)
# 创建表情检测器
detector = EmotionDetector(md_json_path, md_weight_path)
# 执行表情检测并绘制出检测结果
res_img = drawing_image_emotion(test_img, boxes, detector, isCh=True, isDrawEmoji=True)
# 显示出图片
imshow(res_img, size=(12, 12))

png

在测试一个图片

img_2 = cv2.imread('./test_set/test_imgs/02.jpg')
boxes = mtcnnFaceDetector.simpleDetection(img_2, minSizeImage=15)
res_img = drawing_image_emotion(img_2, boxes, detector, isCh=True, isDrawEmoji=True)
imshow(res_img, size=(9, 9))

png

经过测试,两张图像都可以准确预出我们想要的结果,请读者多多尝试其他的数据集,找出模型的问题,并加以修正。

结论

本次主要使用VGGNet-16网络构建一个人脸表情分类的模型,模型较为基础,很多地方还未完善,请读者自行深入研究。

版权声明:如无特殊说明,文章均为本站原创,转载请注明出处

本文链接:http://tunm.top/article/learning_11/