东方耀AI技术分享
标题:
03、模型训练的代码调整一下以及初始训练结果
[打印本页]
作者:
东方耀
时间:
2019-11-7 09:21
标题:
03、模型训练的代码调整一下以及初始训练结果
03、模型训练的代码调整一下以及初始训练结果
dict_keys(['val_loss', 'val_mae', 'loss', 'mae'])
val_loss val_mae loss mae
0 0.009134 0.070623 0.013413 0.073236
1 0.011031 0.067170 0.010952 0.067101
2 0.005622 0.067855 0.010583 0.067400
3 0.005378 0.064024 0.010231 0.066075
4 0.004834 0.062514 0.009657 0.063892
5 0.005470 0.064100 0.009387 0.062929
6 0.003604 0.061187 0.009363 0.063271
7 0.004759 0.067398 0.008875 0.061294
8 0.008716 0.061494 0.008494 0.060054
9 0.009699 0.064915 0.008158 0.058782
10 0.010862 0.062815 0.007335 0.055527
11 0.004432 0.060743 0.007094 0.053928
# -*- coding: utf-8 -*-
__author__ = u'东方耀 微信:dfy_88888'
__date__ = '2019/11/7 08:49'
__product__ = 'PyCharm'
__filename__ = 'train_dfy'
import tensorflow as tf
import numpy as np
import torch as T
from keras.layers import Conv2D, MaxPooling2D, Flatten, PReLU
from keras.layers.core import Dense, Dropout, Activation
from keras.optimizers import SGD, Adam
from keras.models import Model, Sequential
from keras import backend as K
from keras.regularizers import l2
import os.path
import cv2
import skimage.io as iio
import csv
import glob
import pickle
from sklearn.utils import shuffle
from sklearn.model_selection import cross_validate
from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV
import json
from keras import callbacks
import math
import matplotlib.pyplot as plt
from tensorflow import gfile
import pandas as pd
SEED = 666
print(tf.__version__)
print(T.__version__)
def get_model(shape):
"""
预测方向盘角度,以图像为输入,预测方向盘的转动角度
:param shape: 图像尺寸 (128, 128, 3) NHC
:return:
"""
model = Sequential(name='dfy_seq_model')
# 第一层需要指定input_shape 后面不需要
model.add(Conv2D(filters=24, kernel_size=(5, 5), strides=(2, 2),
padding='valid', data_format='channels_last', activation='relu', input_shape=shape))
model.add(Conv2D(filters=36, kernel_size=(5, 5), strides=(2, 2),
padding='valid', data_format='channels_last', activation='relu'))
model.add(Conv2D(filters=48, kernel_size=(5, 5), strides=(2, 2),
padding='valid', data_format='channels_last', activation='relu'))
model.add(Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1),
padding='valid', data_format='channels_last', activation='relu'))
model.add(Conv2D(filters=64, kernel_size=(3, 3), strides=(1, 1),
padding='valid', data_format='channels_last', activation='relu'))
model.add(Flatten(data_format='channels_last'))
model.add(Dense(units=1164, activation='relu'))
model.add(Dense(units=100, activation='relu'))
model.add(Dense(units=50, activation='relu'))
model.add(Dense(units=10, activation='relu'))
# 由于输出的角度是 (-pi/2, pi/2) 要选择好的激活函数
model.add(Dense(units=1, activation='linear'))
# compile: 1、指定优化器 2、损失函数
model.compile(optimizer=Adam(learning_rate=0.001), loss='mse', metrics=['mae'])
return model
# 开始数据增强(基于现有样本数据产生新的更多的训练数据)
def random_brightness(img, degree):
"""
随机调整输入图像的亮度,调整强度于0.1(变黑)和1(无变化)之间
:param img: 输入图像
:param degree: 输入图像对应的转动角度
:return:
"""
return (img, degree)
def horizontal_flip(img, degree):
"""
按照50%的概率水平翻转图像
:param img: 输入图像
:param degree: 输入图像对应的转动角度
:return:
"""
pass
return (img, degree)
def left_right_random_swap(img_address, degree, degree_corr=1.0 / 4):
"""
随机从左、中、右图像中选择一张图像,并相应调整转动的角度
:param img_address: 中间图像的文件路径
:param degree: 中间图像对应的方向盘转动角度
:param degree_corr: 方向盘转动角度调整的值
:return:
"""
return (img_address, degree)
def discard_zero_steering(degrees, rate):
"""
从角度为0的index中随机选择部分index返回
:param degrees: 输入的角度值
:param rate: 丢弃率 rate=0.8 表示80%的index返回
:return:
"""
return degrees
def image_transformation(img_address, degree, data_dir):
# img_address, degree = left_right_random_swap(img_address, degree)
# opencv 读出来的图像是 bgr的
# print('开始读的图片地址(路径有中文):', data_dir+img_address)
img = iio.imread(data_dir + img_address)
# cvt convert 颜色空间的转换
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
# img, degree = random_brightness(img, degree)
# img, degree = horizontal_flip(img, degree)
return (img, degree)
def batch_generator(x, y, batch_size, shape, training=True, data_dir='data/', discard_rate=0.95):
"""
产生批处理的数据Generator 高效读取数据
Data Generator 无需预先生成所有图像增强后的图像,会占用太多的硬盘空间与增加读取硬盘文件所需的时间
:param x: 图像文件路径list
:param y: 方向盘的角度
:param batch_size:
:param shape: 输入图像的尺寸(HWC)
:param training: True时产生训练数据 False时产生validation数据
:param data_dir: 数据目录,包含一个IMG文件夹
:param discard_rate: 随机丢弃角度=0的训练数据的比率
:return:
"""
if training:
x, y = shuffle(x, y)
rand_zero_idx = discard_zero_steering(y, rate=discard_rate)
new_x = np.delete(x, rand_zero_idx, axis=0)
new_y = np.delete(y, rand_zero_idx, axis=0)
else:
new_x = x
new_y = y
offset = 0
while True:
X = np.empty(shape=(batch_size, *shape))
Y = np.empty(shape=(batch_size, 1))
for example in range(batch_size):
img_address, img_steering = new_x[example + offset], new_y[example + offset]
if training:
img, img_steering = image_transformation(img_address, img_steering, data_dir)
else:
img = iio.imread(data_dir + img_address)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
# 先截取图像 后resize 再 归一化到[-0.5, 0.5]
# [NHWC] 四维
X[example, :, :, :] = cv2.resize(img[80:140, 0:320], (shape[0], shape[1])) / 255 - 0.5
Y[example] = img_steering
if (example + 1) + offset > len(new_y) - 1:
# 达到了原来数据的尾部,从头开始
x, y = shuffle(x, y)
rand_zero_idx = discard_zero_steering(y, rate=discard_rate)
new_x = x
new_y = y
new_x = np.delete(new_x, rand_zero_idx, axis=0)
new_y = np.delete(new_y, rand_zero_idx, axis=0)
offset = 0
yield (X, Y)
offset = offset + batch_size
def load_train_datasets(data_path):
with open(data_path + 'driving_log.csv', 'r') as csvfile:
file_reader = csv.reader(csvfile, delimiter=',')
log = []
for row in file_reader:
log.append(row)
log = np.array(log)
# 二维矩阵里面都是字符串 (8037, 7)
print(log.shape)
print(log.dtype)
print(log[:3, :])
# 去掉第一行 表头数据
log = log[1:, :]
ls_imgs = glob.glob(data_path + 'IMG/*.jpg')
# 一共有24108张图片 = 8036*3
print('一共有%d张图片(包括中间、左边、右边)' % len(ls_imgs))
assert len(ls_imgs) == len(log) * 3, 'number of images does not match!'
# 中间摄像头的图片路径 str
x_ = log[:, 0]
# steering str---> float
y_ = log[:, 3].astype(float)
return x_, y_
def plot_learning_curve(history):
# ValueError: arrays must all be same length
# 表格型数据 要求每一列的len一致 这里即:history.history字典里每个key对应的value长度一致
df_history = pd.DataFrame(data=history.history)
print(df_history)
# print(df_history.index)
print(df_history.columns)
# print(df_history.dtypes)
df_history.plot(figsize=(8, 5))
plt.grid(True)
# x就是DataFrame的索引
plt.ylim(0, 1.5)
plt.show()
def train(model, input_shape, X_train, y_train, X_validation, y_validation, data_path):
batch_size = 32
# 使得validation数据集大小为batch_size的整数倍 训练集大小:6428, 验证集大小:1608
num_validation_samples = len(y_validation) - len(y_validation) % batch_size
# num_validation_samples size: 1600
print('num_validation_samples size:', num_validation_samples)
cb_save_best_model = callbacks.ModelCheckpoint(filepath='best_model.h5', monitor='val_loss',
save_best_only=True, mode='min')
# 如果训练连续patience=15(向后看多少步)val_loss did not improve(网络不收敛),提前结束训练
# if (last_loss - current_loss) > min_delta 才算网络是在优化 否则表示loss没有在下降
cb_early_stop = callbacks.EarlyStopping(monitor='val_loss', min_delta=1e-3, patience=5)
cb_tensorboard = callbacks.TensorBoard(log_dir='./TB_Graph')
callbacks_list = [cb_save_best_model, cb_early_stop, cb_tensorboard]
# 开始训练网络 训练集大小:6428, 验证集大小:1608
# steps_per_epoch = 6428 // batch_size =200
history = model.fit_generator(
generator=batch_generator(X_train, y_train, batch_size, input_shape, training=True, data_dir=data_path),
steps_per_epoch=200, validation_steps=num_validation_samples // batch_size,
validation_data=batch_generator(X_validation, y_validation, batch_size, input_shape, training=False, data_dir=data_path),
epochs=20, verbose=1, callbacks=callbacks_list)
# list all data in history
print(history.history.keys())
fig, ax_array = plt.subplots(1, 2)
ax1, ax2 = ax_array
ax1.set_title('model metrics')
ax1.plot(history.history['mae'])
ax1.plot(history.history['val_mae'])
ax1.set_xlabel('epoch')
ax1.set_ylabel('mae')
ax1.legend(['train', 'validation'], loc='upper left')
ax2.set_title('model loss')
ax2.plot(history.history['loss'])
ax2.plot(history.history['val_loss'])
ax2.set_xlabel('epoch')
ax2.set_ylabel('loss')
ax2.legend(['train', 'validation'], loc='upper left')
plt.show()
plot_learning_curve(history)
with open('./trainHistoryDict.pickle', 'wb') as file_pickle:
pickle.dump(history.history, file_pickle)
print('完成 Done!')
def train_model():
data_path = 'F:\\AI_Study_dfy\\项目:自动驾驶之方向盘转动角度预测\data\\'
x_, y_ = load_train_datasets(data_path)
# 使用20%的数据作为测试数据集
test_ratio = 0.2
x_, y_ = shuffle(x_, y_)
X_train, X_validation, y_train, y_validation = train_test_split(x_, y_, random_state=SEED, test_size=test_ratio)
# 训练集大小:6428, 验证集大小:1608 = 8036
print('训练集大小:{}, 验证集大小:{}'.format(len(X_train), len(X_validation)))
# X_train_norm = preprocess_features_dfy(X_train)
# y_train = utils.to_categorical(y_train, n_classes)
input_shape = (128, 128, 3)
model = get_model(input_shape)
# 生成模型结构汇总 挺好!
model.summary()
# 模型保存结构文件
with gfile.GFile('model_structure_dfy.json', 'w') as f:
f.write(model.to_json())
train(model, input_shape, X_train, y_train, X_validation, y_validation, data_path)
if __name__ == '__main__':
train_model()
复制代码
作者:
xsoft
时间:
2020-2-3 15:41
谢谢老师提供的资料。
欢迎光临 东方耀AI技术分享 (http://www.ai111.vip/)
Powered by Discuz! X3.4