05、项目相关数据flickr30k介绍与词表生成_笔记

东方耀

05、项目相关数据flickr30k介绍与词表生成_笔记


训练数据flickr30k（已标注好的图像）下载：链接：https://pan.baidu.com/s/1ola0honCkwPLdV4DW0TGsg
提取码：eggh


预训练好的Inception CNN神经网络数据下载：链接：https://pan.baidu.com/s/1rHivh5dlG6yevwCmY19Y6w
提取码：5ter

1. # -*- coding: utf-8 -*-
   
2. __author__ = u'东方耀 微信：dfy_88888'
   
3. __date__ = '2019/5/15 11:30'
   
4. __product__ = 'PyCharm'
   
5. __filename__ = 'generate_vocab'
   
6. 
   
7. 
   
8. import pprint
   
9. 
   
10. input_description_file = './results_20130124.token'
    
11. output_vocab_file = './vocab.txt'
    
12. 
    
13. 
    
14. def count_vocab(input_description_file):
    
15.     with open(input_description_file, 'r', encoding='utf-8') as f:
    
16.         lines = f.readlines()
    
17.     max_length_of_sentences = 0
    
18.     # key: 句子长度 value: 长度的次数
    
19.     length_dict = {}
    
20.     # key: 词  value: 词频
    
21.     vocab_dict = {}
    
22.     for line in lines:
    
23.         image_id, description = line.strip('\n').split('\t')
    
24.         words = description.strip(' ').split()
    
25.         max_length_of_sentences = max(max_length_of_sentences, len(words))
    
26.         length_dict.setdefault(len(words), 0)
    
27.         length_dict[len(words)] += 1
    
28. 
    
29.         for word in words:
    
30.             vocab_dict.setdefault(word, 0)
    
31.             vocab_dict[word] += 1
    
32.     print('max_length_of_sentences: %d' % max_length_of_sentences)
    
33.     pprint.pprint(length_dict)
    
34. 
    
35.     return vocab_dict
    
36. 
    
37. 
    
38. vocab_dict = count_vocab(input_description_file)
    
39. 
    
40. sorted_vocab_dict = sorted(vocab_dict.items(), key=lambda d: d[1], reverse=True)
    
41. 
    
42. 
    
43. with open(output_vocab_file, 'w', encoding='utf-8') as f:
    
44.     f.write('<UNK>\t99999999999999999\n')
    
45.     for item in sorted_vocab_dict:
    
46.         f.write('%s\t%d\n' % (item[0], item[1]))
    
47. 
    
48. 
    
49. 
    

复制代码