faster rcnn网络结构详解

东方耀

faster rcnn网络结构详解


caffe网络在线可视化工具（需要翻墙）：http://ethereon.github.io/netscope/#/editor

1. name: "VGG_ILSVRC_16_layers"
   
2. 
   
3. input: "data"
   
4. input_shape {
   
5.   dim: 1
   
6.   dim: 3
   
7.   dim: 600
   
8.   dim: 989
   
9. }
   
10. 
    
11. input: "im_info"
    
12. input_shape {
    
13.   dim: 1
    
14.   dim: 3
    
15. }
    
16. 
    
17. layer {
    
18.   name: "conv1_1"
    
19.   type: "Convolution"
    
20.   bottom: "data"
    
21.   top: "conv1_1"
    
22.   convolution_param {
    
23.     num_output: 64
    
24.     pad: 1 kernel_size: 3
    
25.   }
    
26. }
    
27. layer {
    
28.   name: "relu1_1"
    
29.   type: "ReLU"
    
30.   bottom: "conv1_1"
    
31.   top: "conv1_1"
    
32. }
    
33. layer {
    
34.   name: "conv1_2"
    
35.   type: "Convolution"
    
36.   bottom: "conv1_1"
    
37.   top: "conv1_2"
    
38.   convolution_param {
    
39.     num_output: 64
    
40.     pad: 1 kernel_size: 3
    
41.   }
    
42. }
    
43. layer {
    
44.   name: "relu1_2"
    
45.   type: "ReLU"
    
46.   bottom: "conv1_2"
    
47.   top: "conv1_2"
    
48. }
    
49. layer {
    
50.   name: "pool1"
    
51.   type: "Pooling"
    
52.   bottom: "conv1_2"
    
53.   top: "pool1"
    
54.   pooling_param {
    
55.     pool: MAX
    
56.     kernel_size: 2 stride: 2
    
57.   }
    
58. }
    
59. layer {
    
60.   name: "conv2_1"
    
61.   type: "Convolution"
    
62.   bottom: "pool1"
    
63.   top: "conv2_1"
    
64.   convolution_param {
    
65.     num_output: 128
    
66.     pad: 1 kernel_size: 3
    
67.   }
    
68. }
    
69. layer {
    
70.   name: "relu2_1"
    
71.   type: "ReLU"
    
72.   bottom: "conv2_1"
    
73.   top: "conv2_1"
    
74. }
    
75. layer {
    
76.   name: "conv2_2"
    
77.   type: "Convolution"
    
78.   bottom: "conv2_1"
    
79.   top: "conv2_2"
    
80.   convolution_param {
    
81.     num_output: 128
    
82.     pad: 1 kernel_size: 3
    
83.   }
    
84. }
    
85. layer {
    
86.   name: "relu2_2"
    
87.   type: "ReLU"
    
88.   bottom: "conv2_2"
    
89.   top: "conv2_2"
    
90. }
    
91. layer {
    
92.   name: "pool2"
    
93.   type: "Pooling"
    
94.   bottom: "conv2_2"
    
95.   top: "pool2"
    
96.   pooling_param {
    
97.     pool: MAX
    
98.     kernel_size: 2 stride: 2
    
99.   }
    
100. }
     
101. layer {
     
102.   name: "conv3_1"
     
103.   type: "Convolution"
     
104.   bottom: "pool2"
     
105.   top: "conv3_1"
     
106.   convolution_param {
     
107.     num_output: 256
     
108.     pad: 1 kernel_size: 3
     
109.   }
     
110. }
     
111. layer {
     
112.   name: "relu3_1"
     
113.   type: "ReLU"
     
114.   bottom: "conv3_1"
     
115.   top: "conv3_1"
     
116. }
     
117. layer {
     
118.   name: "conv3_2"
     
119.   type: "Convolution"
     
120.   bottom: "conv3_1"
     
121.   top: "conv3_2"
     
122.   convolution_param {
     
123.     num_output: 256
     
124.     pad: 1 kernel_size: 3
     
125.   }
     
126. }
     
127. layer {
     
128.   name: "relu3_2"
     
129.   type: "ReLU"
     
130.   bottom: "conv3_2"
     
131.   top: "conv3_2"
     
132. }
     
133. layer {
     
134.   name: "conv3_3"
     
135.   type: "Convolution"
     
136.   bottom: "conv3_2"
     
137.   top: "conv3_3"
     
138.   convolution_param {
     
139.     num_output: 256
     
140.     pad: 1 kernel_size: 3
     
141.   }
     
142. }
     
143. layer {
     
144.   name: "relu3_3"
     
145.   type: "ReLU"
     
146.   bottom: "conv3_3"
     
147.   top: "conv3_3"
     
148. }
     
149. layer {
     
150.   name: "pool3"
     
151.   type: "Pooling"
     
152.   bottom: "conv3_3"
     
153.   top: "pool3"
     
154.   pooling_param {
     
155.     pool: MAX
     
156.     kernel_size: 2 stride: 2
     
157.   }
     
158. }
     
159. layer {
     
160.   name: "conv4_1"
     
161.   type: "Convolution"
     
162.   bottom: "pool3"
     
163.   top: "conv4_1"
     
164.   convolution_param {
     
165.     num_output: 512
     
166.     pad: 1 kernel_size: 3
     
167.   }
     
168. }
     
169. layer {
     
170.   name: "relu4_1"
     
171.   type: "ReLU"
     
172.   bottom: "conv4_1"
     
173.   top: "conv4_1"
     
174. }
     
175. layer {
     
176.   name: "conv4_2"
     
177.   type: "Convolution"
     
178.   bottom: "conv4_1"
     
179.   top: "conv4_2"
     
180.   convolution_param {
     
181.     num_output: 512
     
182.     pad: 1 kernel_size: 3
     
183.   }
     
184. }
     
185. layer {
     
186.   name: "relu4_2"
     
187.   type: "ReLU"
     
188.   bottom: "conv4_2"
     
189.   top: "conv4_2"
     
190. }
     
191. layer {
     
192.   name: "conv4_3"
     
193.   type: "Convolution"
     
194.   bottom: "conv4_2"
     
195.   top: "conv4_3"
     
196.   convolution_param {
     
197.     num_output: 512
     
198.     pad: 1 kernel_size: 3
     
199.   }
     
200. }
     
201. layer {
     
202.   name: "relu4_3"
     
203.   type: "ReLU"
     
204.   bottom: "conv4_3"
     
205.   top: "conv4_3"
     
206. }
     
207. layer {
     
208.   name: "pool4"
     
209.   type: "Pooling"
     
210.   bottom: "conv4_3"
     
211.   top: "pool4"
     
212.   pooling_param {
     
213.     pool: MAX
     
214.     kernel_size: 2 stride: 2
     
215.   }
     
216. }
     
217. layer {
     
218.   name: "conv5_1"
     
219.   type: "Convolution"
     
220.   bottom: "pool4"
     
221.   top: "conv5_1"
     
222.   convolution_param {
     
223.     num_output: 512
     
224.     pad: 1 kernel_size: 3
     
225.   }
     
226. }
     
227. layer {
     
228.   name: "relu5_1"
     
229.   type: "ReLU"
     
230.   bottom: "conv5_1"
     
231.   top: "conv5_1"
     
232. }
     
233. layer {
     
234.   name: "conv5_2"
     
235.   type: "Convolution"
     
236.   bottom: "conv5_1"
     
237.   top: "conv5_2"
     
238.   convolution_param {
     
239.     num_output: 512
     
240.     pad: 1 kernel_size: 3
     
241.   }
     
242. }
     
243. layer {
     
244.   name: "relu5_2"
     
245.   type: "ReLU"
     
246.   bottom: "conv5_2"
     
247.   top: "conv5_2"
     
248. }
     
249. layer {
     
250.   name: "conv5_3"
     
251.   type: "Convolution"
     
252.   bottom: "conv5_2"
     
253.   top: "conv5_3"
     
254.   convolution_param {
     
255.     num_output: 512
     
256.     pad: 1 kernel_size: 3
     
257.   }
     
258. }
     
259. layer {
     
260.   name: "relu5_3"
     
261.   type: "ReLU"
     
262.   bottom: "conv5_3"
     
263.   top: "conv5_3"
     
264. }
     
265. 
     
266. #========= RPN ============
     
267. 
     
268. layer {
     
269.   name: "rpn_conv/3x3"
     
270.   type: "Convolution"
     
271.   bottom: "conv5_3"
     
272.   top: "rpn/output"
     
273.   convolution_param {
     
274.     num_output: 512
     
275.     kernel_size: 3 pad: 1 stride: 1
     
276.   }
     
277. }
     
278. layer {
     
279.   name: "rpn_relu/3x3"
     
280.   type: "ReLU"
     
281.   bottom: "rpn/output"
     
282.   top: "rpn/output"
     
283. }
     
284. 
     
285. layer {
     
286.   name: "rpn_cls_score"
     
287.   type: "Convolution"
     
288.   bottom: "rpn/output"
     
289.   top: "rpn_cls_score"
     
290.   convolution_param {
     
291.     num_output: 18   # 2(bg/fg) * 9(anchors)
     
292.     kernel_size: 1 pad: 0 stride: 1
     
293.   }
     
294. }
     
295. layer {
     
296.   name: "rpn_bbox_pred"
     
297.   type: "Convolution"
     
298.   bottom: "rpn/output"
     
299.   top: "rpn_bbox_pred"
     
300.   convolution_param {
     
301.     num_output: 36   # 4 * 9(anchors)
     
302.     kernel_size: 1 pad: 0 stride: 1
     
303.   }
     
304. }
     
305. layer {
     
306.    bottom: "rpn_cls_score"
     
307.    top: "rpn_cls_score_reshape"
     
308.    name: "rpn_cls_score_reshape"
     
309.    type: "Reshape"
     
310.    reshape_param { shape { dim: 0 dim: 2 dim: -1 dim: 0 } }
     
311. }
     
312. 
     
313. #========= RoI Proposal ============
     
314. 
     
315. layer {
     
316.   name: "rpn_cls_prob"
     
317.   type: "Softmax"
     
318.   bottom: "rpn_cls_score_reshape"
     
319.   top: "rpn_cls_prob"
     
320. }
     
321. layer {
     
322.   name: 'rpn_cls_prob_reshape'
     
323.   type: 'Reshape'
     
324.   bottom: 'rpn_cls_prob'
     
325.   top: 'rpn_cls_prob_reshape'
     
326.   reshape_param { shape { dim: 0 dim: 18 dim: -1 dim: 0 } }
     
327. }
     
328. layer {
     
329.   name: 'proposal'
     
330.   type: 'Python'
     
331.   bottom: 'rpn_cls_prob_reshape'
     
332.   bottom: 'rpn_bbox_pred'
     
333.   bottom: 'im_info'
     
334.   top: 'rois'
     
335.   python_param {
     
336.     module: 'rpn.proposal_layer'
     
337.     layer: 'ProposalLayer'
     
338.     param_str: "'feat_stride': 16"
     
339.   }
     
340. }
     
341. 
     
342. #========= RCNN ============
     
343. 
     
344. layer {
     
345.   name: "roi_pool5"
     
346.   type: "ROIPooling"
     
347.   bottom: "conv5_3"
     
348.   bottom: "rois"
     
349.   top: "pool5"
     
350.   roi_pooling_param {
     
351.     pooled_w: 7
     
352.     pooled_h: 7
     
353.     spatial_scale: 0.0625 # 1/16
     
354.   }
     
355. }
     
356. layer {
     
357.   name: "fc6"
     
358.   type: "InnerProduct"
     
359.   bottom: "pool5"
     
360.   top: "fc6"
     
361.   inner_product_param {
     
362.     num_output: 4096
     
363.   }
     
364. }
     
365. layer {
     
366.   name: "relu6"
     
367.   type: "ReLU"
     
368.   bottom: "fc6"
     
369.   top: "fc6"
     
370. }
     
371. layer {
     
372.   name: "fc7"
     
373.   type: "InnerProduct"
     
374.   bottom: "fc6"
     
375.   top: "fc7"
     
376.   inner_product_param {
     
377.     num_output: 4096
     
378.   }
     
379. }
     
380. layer {
     
381.   name: "relu7"
     
382.   type: "ReLU"
     
383.   bottom: "fc7"
     
384.   top: "fc7"
     
385. }
     
386. layer {
     
387.   name: "cls_score"
     
388.   type: "InnerProduct"
     
389.   bottom: "fc7"
     
390.   top: "cls_score"
     
391.   inner_product_param {
     
392.     num_output: 21
     
393.   }
     
394. }
     
395. layer {
     
396.   name: "bbox_pred"
     
397.   type: "InnerProduct"
     
398.   bottom: "fc7"
     
399.   top: "bbox_pred"
     
400.   inner_product_param {
     
401.     num_output: 84
     
402.   }
     
403. }
     
404. layer {
     
405.   name: "cls_prob"
     
406.   type: "Softmax"
     
407.   bottom: "cls_score"
     
408.   top: "cls_prob"
     
409. }
     

复制代码

层名+shape：data        (1, 3, 600, 989)
层名+shape：im_info        (1, 3)
层名+shape：conv1_1        (1, 64, 600, 989)
层名+shape：conv1_2        (1, 64, 600, 989)
层名+shape：pool1        (1, 64, 300, 495)
层名+shape：conv2_1        (1, 128, 300, 495)
层名+shape：conv2_2        (1, 128, 300, 495)
层名+shape：pool2        (1, 128, 150, 248)
层名+shape：conv3_1        (1, 256, 150, 248)
层名+shape：conv3_2        (1, 256, 150, 248)
层名+shape：conv3_3        (1, 256, 150, 248)
层名+shape：pool3        (1, 256, 75, 124)
层名+shape：conv4_1        (1, 512, 75, 124)
层名+shape：conv4_2        (1, 512, 75, 124)
层名+shape：conv4_3        (1, 512, 75, 124)
层名+shape：pool4        (1, 512, 38, 62)
层名+shape：conv5_1        (1, 512, 38, 62)
层名+shape：conv5_2        (1, 512, 38, 62)
层名+shape：conv5_3        (1, 512, 38, 62)
层名+shape：conv5_3_relu5_3_0_split_0        (1, 512, 38, 62)
层名+shape：conv5_3_relu5_3_0_split_1        (1, 512, 38, 62)
层名+shape：rpn/output        (1, 512, 38, 62)
层名+shape：rpn/output_rpn_relu/3x3_0_split_0        (1, 512, 38, 62)
层名+shape：rpn/output_rpn_relu/3x3_0_split_1        (1, 512, 38, 62)
层名+shape：rpn_cls_score        (1, 18, 38, 62)
层名+shape：rpn_bbox_pred        (1, 36, 38, 62)
层名+shape：rpn_cls_score_reshape        (1, 2, 342, 62)
层名+shape：rpn_cls_prob        (1, 2, 342, 62)
层名+shape：rpn_cls_prob_reshape        (1, 18, 38, 62)
层名+shape：rois        (300, 5)
层名+shape：pool5        (300, 512, 7, 7)
层名+shape：fc6        (300, 4096)
层名+shape：fc7        (300, 4096)
层名+shape：fc7_relu7_0_split_0        (300, 4096)
层名+shape：fc7_relu7_0_split_1        (300, 4096)
层名+shape：cls_score        (300, 21)
层名+shape：bbox_pred        (300, 84)
层名+shape：cls_prob        (300, 21)




层名+网络W与b：conv1_1        (64, 3, 3, 3) (64,)
层名+网络W与b：conv1_2        (64, 64, 3, 3) (64,)
层名+网络W与b：conv2_1        (128, 64, 3, 3) (128,)
层名+网络W与b：conv2_2        (128, 128, 3, 3) (128,)
层名+网络W与b：conv3_1        (256, 128, 3, 3) (256,)
层名+网络W与b：conv3_2        (256, 256, 3, 3) (256,)
层名+网络W与b：conv3_3        (256, 256, 3, 3) (256,)
层名+网络W与b：conv4_1        (512, 256, 3, 3) (512,)
层名+网络W与b：conv4_2        (512, 512, 3, 3) (512,)
层名+网络W与b：conv4_3        (512, 512, 3, 3) (512,)
层名+网络W与b：conv5_1        (512, 512, 3, 3) (512,)
层名+网络W与b：conv5_2        (512, 512, 3, 3) (512,)
层名+网络W与b：conv5_3        (512, 512, 3, 3) (512,)
层名+网络W与b：rpn_conv/3x3        (512, 512, 3, 3) (512,)
层名+网络W与b：rpn_cls_score        (18, 512, 1, 1) (18,)
层名+网络W与b：rpn_bbox_pred        (36, 512, 1, 1) (36,)
层名+网络W与b：fc6        (4096, 25088) (4096,)
层名+网络W与b：fc7        (4096, 4096) (4096,)
层名+网络W与b：cls_score        (21, 4096) (21,)
层名+网络W与b：bbox_pred        (84, 4096) (84,)
前向计算结果blobs_out:
['bbox_pred', 'cls_prob'] (300, 84) (300, 21)

关键层的分析：
layer {
  name: 'proposal'
  type: 'Python'
  bottom: 'rpn_cls_prob_reshape'
  bottom: 'rpn_bbox_pred'
  bottom: 'im_info'
  top: 'rois'
  python_param {
    module: 'rpn.proposal_layer'
    layer: 'ProposalLayer'
    param_str: "'feat_stride': 16"
  }
}
输入是：
rpn_cls_prob_reshape        (1, 18, 38, 62)
rpn_bbox_pred        (1, 36, 38, 62)
im_info        (1, 3)
输出是：
rois        (300, 5)

class ProposalLayer(caffe.Layer):里面有生成9种anchor box

generate_anchors(base_size=16, ratios=[0.5, 1, 2],
                     scales=np.array([8, 16, 32]))

ratios就是高宽比H/W(或 长宽比)   ：
    # ws 宽度是缩放后面积的开方 因为不一定是矩阵
    ws = np.round(np.sqrt(size_ratios))
    # 高度=宽度×缩放比例 ratios就是高宽比  np.round是四舍五入
    hs = np.round(ws * ratios)
scales是缩放比例：
ws = w * scales
hs = h * scales

layer {
  name: "roi_pool5"
  type: "ROIPooling"
  bottom: "conv5_3"
  bottom: "rois"
  top: "pool5"
  roi_pooling_param {
    pooled_w: 7
    pooled_h: 7
    spatial_scale: 0.0625 # 1/16
  }
}输入：
conv5_3        (1, 512, 38, 62)
rois        (300, 5)
输出：
pool5        (300, 512, 7, 7)
ROI pooling总结：
（1）用于目标检测任务；
（2）允许我们对CNN中的feature map进行reuse；
（3）可以显著加速training和testing速度；
（4）允许end-to-end的形式训练目标检测系统。

xsoft

谢谢老师提供的资料。