07、神经网络的超参数搜索

东方耀

07、神经网络的超参数搜索




# 搜索学习率这个超参：[1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2]
手动搜索：需要用多层for循环 同时每次只能搜索一组超参 无法并行化

1. # 搜索学习率这个超参：[1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2]
   
2. learning_rates = [1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2]
   
3. historys = []
   
4. test_losses = []
   
5. for lr in learning_rates:
   
6.     model = keras.models.Sequential(layers=[
   
7.         keras.layers.Dense(30, activation='relu', input_shape=(x_train.shape[1], )),
   
8.         keras.layers.Dense(1)
   
9.     ], name='model_dfy')
   
10. 
    
11.     model.compile(optimizer=keras.optimizers.Adam(lr), loss='mean_squared_error')
    
12.     callbacks = [
    
13.         keras.callbacks.EarlyStopping(patience=5, min_delta=1e-2)
    
14.     ]
    
15.     history = model.fit(x_train_scaled, y_train, validation_data=(x_valid_scaled, y_valid), epochs=50,
    
16.                         callbacks=callbacks)
    
17.     # Returns the loss value & metrics values for the model in test mode.
    
18.     test_losses.append(model.evaluate(x_test_scaled, y_test))
    
19.     historys.append(history)

复制代码

sklearn封装keras模型Demo:

1. import numpy as np
   
2. import matplotlib as mpl
   
3. import matplotlib.pyplot as plt
   
4. import sklearn
   
5. import pandas as pd
   
6. import os
   
7. import sys
   
8. import time
   
9. import tensorflow as tf
   
10. from tensorflow import keras
    
11. 
    
12. print(sys.version_info)
    
13. for module in mpl, np, pd, sklearn, tf, keras:
    
14.     print(module.__name__, module.__version__)
    
15. 
    
16. from sklearn.datasets import fetch_california_housing
    
17. 
    
18. housing = fetch_california_housing()
    
19. # print(housing.DESCR)
    
20. print(housing.data.shape)
    
21. print(housing.target.shape)
    
22. 
    
23. from sklearn.model_selection import train_test_split
    
24. 
    
25. x_train_all, x_test, y_train_all, y_test = train_test_split(
    
26.     housing.data, housing.target, random_state=7)
    
27. x_train, x_valid, y_train, y_valid = train_test_split(
    
28.     x_train_all, y_train_all, random_state=11)
    
29. print(x_train.shape, y_train.shape)
    
30. print(x_valid.shape, y_valid.shape)
    
31. print(x_test.shape, y_test.shape)
    
32. 
    
33. from sklearn.preprocessing import StandardScaler
    
34. 
    
35. scaler = StandardScaler()
    
36. x_train_scaled = scaler.fit_transform(x_train)
    
37. x_valid_scaled = scaler.transform(x_valid)
    
38. x_test_scaled = scaler.transform(x_test)
    
39. 
    
40. 
    
41. # 使用sklearn中RandomizedSearchCV
    
42. # 1、sklearn封装keras模型
    
43. # 2、定义超参集合
    
44. # 3、搜索超参
    
45. 
    
46. def build_model(hidden_layer=2, layer_size=30, learning_rate=3e-3):
    
47.     model = keras.models.Sequential(name='model_for_sklearn')
    
48.     model.add(keras.layers.Dense(layer_size, activation='relu', input_shape=(x_train.shape[1], )))
    
49.     for _ in range(hidden_layer-1):
    
50.         model.add(keras.layers.Dense(layer_size, activation='relu'))
    
51.     model.add(keras.layers.Dense(1))
    
52.     optimizer = keras.optimizers.Adam(learning_rate)
    
53.     model.compile(optimizer, loss='mse')
    
54.     model.summary()
    
55.     return model
    
56. 
    
57. 
    
58. # Implementation of the scikit-learn regressor API for Keras
    
59. # 注意 build_fn= 函数名
    
60. sklearn_model = keras.wrappers.scikit_learn.KerasRegressor(build_fn=build_model)
    
61. # sklearn_model.summary()
    
62. callbacks = [
    
63.     keras.callbacks.EarlyStopping(patience=5, min_delta=1e-2)
    
64. ]
    
65. 
    
66. history = sklearn_model.fit(x_train_scaled, y_train, validation_data=(x_valid_scaled, y_valid), epochs=50,
    
67.                     callbacks=callbacks)
    
68. 
    
69. 
    
70. def plot_learning_curve(history):
    
71.     # ValueError: arrays must all be same length
    
72.     # 表格型数据 要求每一列的len一致 这里即：history.history字典里每个key对应的value长度一致
    
73.     df_history = pd.DataFrame(data=history.history)
    
74.     print(df_history)
    
75.     df_history.plot(figsize=(8, 5))
    
76.     plt.grid(True)
    
77.     # x就是DataFrame的索引
    
78.     plt.ylim(0, 1)
    
79.     plt.show()
    
80. 
    
81. 
    
82. plot_learning_curve(history)
    
83. 
    
84. from sklearn.metrics import mean_squared_error
    
85. # Returns the loss value & metrics values for the model in test mode.
    
86. # print(sklearn_model.score(x_test_scaled, y_test))
    
87. print(mean_squared_error(y_test, sklearn_model.predict(x_test_scaled)))

复制代码

使用sklearn(封装keras模型)中RandomizedSearchCV示例：

1. import numpy as np
   
2. import matplotlib as mpl
   
3. import matplotlib.pyplot as plt
   
4. import sklearn
   
5. import pandas as pd
   
6. import os
   
7. import sys
   
8. import time
   
9. import tensorflow as tf
   
10. from tensorflow import keras
    
11. 
    
12. print(sys.version_info)
    
13. for module in mpl, np, pd, sklearn, tf, keras:
    
14.     print(module.__name__, module.__version__)
    
15. 
    
16. from sklearn.datasets import fetch_california_housing
    
17. 
    
18. housing = fetch_california_housing()
    
19. # print(housing.DESCR)
    
20. print(housing.data.shape)
    
21. print(housing.target.shape)
    
22. 
    
23. from sklearn.model_selection import train_test_split
    
24. 
    
25. x_train_all, x_test, y_train_all, y_test = train_test_split(
    
26.     housing.data, housing.target, random_state=7)
    
27. x_train, x_valid, y_train, y_valid = train_test_split(
    
28.     x_train_all, y_train_all, random_state=11)
    
29. print(x_train.shape, y_train.shape)
    
30. print(x_valid.shape, y_valid.shape)
    
31. print(x_test.shape, y_test.shape)
    
32. 
    
33. from sklearn.preprocessing import StandardScaler
    
34. 
    
35. scaler = StandardScaler()
    
36. x_train_scaled = scaler.fit_transform(x_train)
    
37. x_valid_scaled = scaler.transform(x_valid)
    
38. x_test_scaled = scaler.transform(x_test)
    
39. 
    
40. 
    
41. # 使用sklearn中RandomizedSearchCV, GridSearchCV
    
42. # 1、sklearn封装keras模型
    
43. # 2、定义超参集合
    
44. # 3、搜索超参
    
45. 
    
46. def build_keras_model(hidden_layer=2, layer_size=30, learning_rate=3e-3):
    
47.     model = keras.models.Sequential(name='model_for_sklearn')
    
48.     model.add(keras.layers.Dense(layer_size, activation='relu', input_shape=(x_train.shape[1], )))
    
49.     for _ in range(hidden_layer-1):
    
50.         model.add(keras.layers.Dense(layer_size, activation='relu'))
    
51.     model.add(keras.layers.Dense(1))
    
52.     optimizer = keras.optimizers.Adam(learning_rate)
    
53.     model.compile(optimizer, loss='mse')
    
54.     model.summary()
    
55.     return model
    
56. 
    
57. 
    
58. # Implementation of the scikit-learn regressor API for Keras
    
59. # 注意 build_fn= 函数名
    
60. sklearn_model = keras.wrappers.scikit_learn.KerasRegressor(build_fn=build_keras_model)
    
61. # sklearn_model.summary()
    
62. 
    
63. 
    
64. from sklearn.model_selection import RandomizedSearchCV, GridSearchCV
    
65. # 参数网格
    
66. param_grid = [
    
67.     {
    
68.         'hidden_layer': [2, 3, 4, 5, 6],
    
69.         'layer_size': np.arange(30, 100),
    
70.         'learning_rate': [1e-4, 3e-4, 1e-3, 3e-3, 1e-2, 3e-2]
    
71.     }
    
72. ]
    
73. 
    
74. # grid_search = GridSearchCV(estimator=sklearn_model, param_grid=param_grid, n_jobs=1, cv=5, verbose=1)
    
75. # grid_search.fit(x_train_scaled, y_train)
    
76. # print(grid_search.best_params_)
    
77. # print(grid_search.best_score_)
    
78. 
    
79. from scipy.stats import reciprocal
    
80. # 参数分布
    
81. param_distributions = {
    
82.         'hidden_layer': [2, 3, 4, 5, 6],
    
83.         'layer_size': np.arange(30, 100),
    
84.         'learning_rate': reciprocal(1e-4, 1e-2)
    
85. }
    
86. 
    
87. # estimator 推算子  n_iter=10 随机生成10个候选candidates参数集合
    
88. # Fitting 5 folds for each of 10 candidates, totalling 50 fits
    
89. random_search = RandomizedSearchCV(estimator=sklearn_model, param_distributions=param_distributions,
    
90.                                    n_iter=10, n_jobs=1, cv=5, verbose=1, random_state=666)
    
91. # 报错：PicklingError: Could not pickle the task to send it to the workers. 原因：无法并行化 设置n_jobs=1
    
92. # random_search.fit(x_train_scaled, y_train)
    
93. 
    
94. callbacks = [
    
95.     keras.callbacks.EarlyStopping(patience=3, min_delta=1e-2)
    
96. ]
    
97. random_search.fit(x_train_scaled, y_train, validation_data=(x_valid_scaled, y_valid), epochs=50,
    
98.                     callbacks=callbacks)
    
99. print(random_search.best_params_)
    
100. # {'hidden_layer': 6, 'layer_size': 32, 'learning_rate': 0.0048794771174957945}
     
101. print(random_search.best_score_)
     
102. print(random_search.best_estimator_)
     
103. best_sklearn_model = random_search.best_estimator_
     
104. 
     
105. 
     
106. def plot_learning_curve(history):
     
107.     # ValueError: arrays must all be same length
     
108.     # 表格型数据 要求每一列的len一致 这里即：history.history字典里每个key对应的value长度一致
     
109.     df_history = pd.DataFrame(data=history.history)
     
110.     print(df_history)
     
111.     df_history.plot(figsize=(8, 5))
     
112.     plt.grid(True)
     
113.     # x就是DataFrame的索引
     
114.     plt.ylim(0, 1)
     
115.     plt.show()
     
116. 
     
117. 
     
118. # plot_learning_curve(history)
     
119. 
     
120. from sklearn.metrics import mean_squared_error
     
121. print(best_sklearn_model.score(x_test_scaled, y_test))
     
122. print(mean_squared_error(y_test, best_sklearn_model.predict(x_test_scaled)))
     
123. 
     
124. best_keras_model = best_sklearn_model.model
     
125. # # Returns the loss value & metrics values for the model in test mode.
     
126. print(best_keras_model.evaluate(x_test_scaled, y_test))

复制代码

使用sklearn(封装keras模型)中GridSearchCV示例：

1. import numpy as np
   
2. import matplotlib as mpl
   
3. import matplotlib.pyplot as plt
   
4. import sklearn
   
5. import pandas as pd
   
6. import os
   
7. import sys
   
8. import time
   
9. import tensorflow as tf
   
10. from tensorflow import keras
    
11. 
    
12. print(sys.version_info)
    
13. for module in mpl, np, pd, sklearn, tf, keras:
    
14.     print(module.__name__, module.__version__)
    
15. 
    
16. from sklearn.datasets import fetch_california_housing
    
17. 
    
18. housing = fetch_california_housing()
    
19. # print(housing.DESCR)
    
20. print(housing.data.shape)
    
21. print(housing.target.shape)
    
22. 
    
23. from sklearn.model_selection import train_test_split
    
24. 
    
25. x_train_all, x_test, y_train_all, y_test = train_test_split(
    
26.     housing.data, housing.target, random_state=7)
    
27. x_train, x_valid, y_train, y_valid = train_test_split(
    
28.     x_train_all, y_train_all, random_state=11)
    
29. print(x_train.shape, y_train.shape)
    
30. print(x_valid.shape, y_valid.shape)
    
31. print(x_test.shape, y_test.shape)
    
32. 
    
33. from sklearn.preprocessing import StandardScaler
    
34. 
    
35. scaler = StandardScaler()
    
36. x_train_scaled = scaler.fit_transform(x_train)
    
37. x_valid_scaled = scaler.transform(x_valid)
    
38. x_test_scaled = scaler.transform(x_test)
    
39. 
    
40. 
    
41. # 使用sklearn中RandomizedSearchCV, GridSearchCV
    
42. # 1、sklearn封装keras模型
    
43. # 2、定义超参集合
    
44. # 3、搜索超参
    
45. 
    
46. def build_keras_model(hidden_layer=2, layer_size=30, learning_rate=3e-3):
    
47.     model = keras.models.Sequential(name='model_for_sklearn')
    
48.     model.add(keras.layers.Dense(layer_size, activation='relu', input_shape=(x_train.shape[1], )))
    
49.     for _ in range(hidden_layer-1):
    
50.         model.add(keras.layers.Dense(layer_size, activation='relu'))
    
51.     model.add(keras.layers.Dense(1))
    
52.     optimizer = keras.optimizers.Adam(learning_rate)
    
53.     model.compile(optimizer, loss='mse')
    
54.     model.summary()
    
55.     return model
    
56. 
    
57. 
    
58. # Implementation of the scikit-learn regressor API for Keras
    
59. # 注意 build_fn= 函数名
    
60. sklearn_model = keras.wrappers.scikit_learn.KerasRegressor(build_fn=build_keras_model)
    
61. # sklearn_model.summary()
    
62. 
    
63. 
    
64. from sklearn.model_selection import GridSearchCV
    
65. # 参数网格
    
66. param_grid = [
    
67.     {
    
68.         'hidden_layer': [3, 4, 5, 6],
    
69.         'layer_size': [30, 35, 40],
    
70.         'learning_rate': [1e-4, 3e-4, 1e-3, 3e-3, 1e-2]
    
71.     }
    
72. ]
    
73. # Fitting 3 folds for each of 60 candidates, totalling 180 fits
    
74. grid_search = GridSearchCV(estimator=sklearn_model, param_grid=param_grid, n_jobs=1, cv=3, verbose=1)
    
75. 
    
76. # 报错：PicklingError: Could not pickle the task to send it to the workers. 原因：无法并行化 设置n_jobs=1
    
77. # grid_search.fit(x_train_scaled, y_train)
    
78. 
    
79. callbacks = [
    
80.     keras.callbacks.EarlyStopping(patience=3, min_delta=1e-2)
    
81. ]
    
82. grid_search.fit(x_train_scaled, y_train, validation_data=(x_valid_scaled, y_valid), epochs=50,
    
83.                     callbacks=callbacks)
    
84. print(grid_search.best_params_)
    
85. # {'hidden_layer': 6, 'layer_size': 40, 'learning_rate': 0.001}
    
86. print(grid_search.best_score_)
    
87. print(grid_search.best_estimator_)
    
88. best_sklearn_model = grid_search.best_estimator_
    
89. 
    
90. 
    
91. def plot_learning_curve(history):
    
92.     # ValueError: arrays must all be same length
    
93.     # 表格型数据 要求每一列的len一致 这里即：history.history字典里每个key对应的value长度一致
    
94.     df_history = pd.DataFrame(data=history.history)
    
95.     print(df_history)
    
96.     df_history.plot(figsize=(8, 5))
    
97.     plt.grid(True)
    
98.     # x就是DataFrame的索引
    
99.     plt.ylim(0, 1)
    
100.     plt.show()
     
101. 
     
102. 
     
103. history = best_sklearn_model.fit(x_train_scaled, y_train, validation_data=(x_valid_scaled, y_valid), epochs=50,
     
104.                     callbacks=callbacks)
     
105. 
     
106. plot_learning_curve(history)
     
107. 
     
108. 
     
109. from sklearn.metrics import mean_squared_error
     
110. print(best_sklearn_model.score(x_test_scaled, y_test))
     
111. print(mean_squared_error(y_test, best_sklearn_model.predict(x_test_scaled)))
     
112. 
     
113. best_keras_model = best_sklearn_model.model
     
114. # # Returns the loss value & metrics values for the model in test mode.
     
115. print(best_keras_model.evaluate(x_test_scaled, y_test))

复制代码

东方老师AI官网：http://www.ai111.vip
有任何问题可联系东方老师微信：dfy_88888
【微信二维码图片】