04、项目开发的一般流程与数据预处理_笔记与视频

东方耀

04、项目开发的一般流程与数据预处理_笔记与视频

课程配套的数据cnews_data.zip下载【回复本帖可见】：

游客，如果您要查看本帖隐藏内容请回复

高清视频下载地址【回复本帖可见】：

游客，如果您要查看本帖隐藏内容请回复

1. # -*- coding: utf-8 -*-
   
2. __author__ = u'东方耀 微信：dfy_88888'
   
3. __date__ = '2019/3/21 20:28'
   
4. __product__ = 'PyCharm'
   
5. __filename__ = '1_pre_process'
   
6. 
   
7. 
   
8. import sys
   
9. import os
   
10. import jieba
    
11. 
    
12. # input file
    
13. train_file = './cnews_data/cnews.train.txt'
    
14. test_file = './cnews_data/cnews.test.txt'
    
15. val_file = './cnews_data/cnews.val.txt'
    
16. 
    
17. 
    
18. # output file 分词后的
    
19. seg_train_file = './cnews_data/cnews.train.seg.txt'
    
20. seg_test_file = './cnews_data/cnews.test.seg.txt'
    
21. seg_val_file = './cnews_data/cnews.val.seg.txt'
    
22. 
    
23. 
    
24. # 词表文件
    
25. vocab_file = './cnews_data/cnews.vocab.txt'
    
26. 
    
27. # 类别文件
    
28. category_file = './cnews_data/cnews.category.txt'
    
29. 
    
30. with open(train_file, 'r', encoding='utf-8') as f:
    
31.     lines = f.readlines()
    
32. 
    
33. # label, content = lines[10550].strip('\r\n').split('\t')
    
34. # print(label)
    
35. # print(content)
    
36. 
    
37. # word_iter = jieba.cut(content)
    
38. # print('/'.join(word_iter))
    
39. 
    
40. 
    
41. def generate_seg_file(input_file, output_seg_file):
    
42.     with open(input_file, 'r', encoding='utf-8') as f:
    
43.         lines = f.readlines()
    
44.     with open(output_seg_file, 'w', encoding='utf-8') as f:
    
45.         for line in lines:
    
46.             label, content = line.strip('\r\n').split('\t')
    
47.             word_iter = jieba.cut(content)
    
48.             word_content = ''
    
49.             for word in word_iter:
    
50.                 word = word.strip(' ')
    
51.                 if word != '':
    
52.                     word_content += word + ' '
    
53.             out_line = '%s\t%s\n' % (label, word_content.strip(' '))
    
54.             f.write(out_line)
    
55. 
    
56. 
    
57. # generate_seg_file(val_file, seg_val_file)
    
58. # generate_seg_file(train_file, seg_train_file)
    
59. # generate_seg_file(test_file, seg_test_file)
    
60. 
    
61. 
    
62. def generate_vocab_file(input_seg_file, output_vocab_file):
    
63.     with open(input_seg_file, 'r', encoding='utf-8') as f:
    
64.         lines = f.readlines()
    
65.     # dict {key, value}  word:frequency
    
66.     word_dict = {}
    
67.     for line in lines:
    
68.         label, content = line.strip('\r\n').split('\t')
    
69.         for word in content.split(' '):
    
70.             word_dict.setdefault(word, 0)
    
71.             word_dict[word] += 1
    
72.     # sorted_word_dict [(word, frequency), (), ()]
    
73.     sorted_word_dict = sorted(word_dict.items(), key=lambda d: d[1], reverse=True)
    
74.     with open(output_vocab_file, 'w', encoding='utf-8') as f:
    
75.         f.write('<UNK>\t999999999999\n')
    
76.         for item in sorted_word_dict:
    
77.             f.write('%s\t%s\n' % (item[0], item[1]))
    
78. 
    
79. 
    
80. # generate_vocab_file(seg_train_file, vocab_file)
    
81. 
    
82. 
    
83. def generate_category_file(input_file, output_category_file):
    
84.     with open(input_file, 'r', encoding='utf-8') as f:
    
85.         lines = f.readlines()
    
86.     category_dict = {}
    
87.     for line in lines:
    
88.         label, content = line.strip('\r\n').split('\t')
    
89.         category_dict.setdefault(label, 0)
    
90.         category_dict[label] += 1
    
91.     category_number = len(category_dict)
    
92.     with open(output_category_file, 'w', encoding='utf-8') as f:
    
93.         for category in category_dict.keys():
    
94.             out_line = '%s\n' % category
    
95.             print('%s\t%d\n' % (category, category_dict[category]))
    
96.             f.write(out_line)
    
97. 
    
98. 
    
99. generate_category_file(seg_train_file, category_file)
    
100. 
     
101. 
     
102. 
     
103. 
     
104. 
     
105. 
     
106. 
     
107. 
     
108. 
     
109. 
     

复制代码

lovestudy

4

weiguangnixia

发生的范德萨范德萨

zjshuai

grzt grzt grzt grzt

m379896771

04、项目开发的一般流程与数据预处理_笔记与视频

阳阳11

谢谢楼主分享

jlbu

1111

Sy赵小赵

1

Sybarite

1111

何马

1111