Skip to content

Commit 73c8b4c

Browse files
Echo9573Echo9573
committed
the 8th chapter
1 parent 842f18b commit 73c8b4c

13 files changed

Lines changed: 2156 additions & 0 deletions

‎chapter8/.ipynb_checkpoints/1_1dataPreprocess-checkpoint.ipynb‎

Lines changed: 399 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,399 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 1,
6+
"metadata": {},
7+
"outputs": [
8+
{
9+
"name": "stdout",
10+
"output_type": "stream",
11+
"text": [
12+
"正在进行热毒蕴结证型系数的聚类\n"
13+
]
14+
},
15+
{
16+
"name": "stderr",
17+
"output_type": "stream",
18+
"text": [
19+
"D:\\Anaconda2\\lib\\site-packages\\ipykernel_launcher.py:42: FutureWarning: pd.rolling_mean is deprecated for Series and will be removed in a future version, replace with \n",
20+
"\tSeries.rolling(window=2,center=False).mean()\n",
21+
"D:\\Anaconda2\\lib\\site-packages\\ipykernel_launcher.py:43: SettingWithCopyWarning: \n",
22+
"A value is trying to be set on a copy of a slice from a DataFrame\n",
23+
"\n",
24+
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
25+
]
26+
},
27+
{
28+
"name": "stdout",
29+
"output_type": "stream",
30+
"text": [
31+
"正在进行冲任失调证型系数的聚类\n",
32+
"正在进行肝肾阴虚证型系数的聚类\n",
33+
"正在进行气血两虚证型系数的聚类\n",
34+
"正在进行肝气郁结证型系数的聚类\n",
35+
"正在进行脾胃虚弱证型系数的聚类\n",
36+
" 1 2 3 4\n",
37+
"A 0.0 0.178698 0.257724 0.351843\n",
38+
"An 240.0 356.000000 281.000000 53.000000\n",
39+
"B 0.0 0.147923 0.287039 0.459367\n",
40+
"Bn 316.0 394.000000 174.000000 46.000000\n",
41+
"C 0.0 0.202149 0.289061 0.423537\n",
42+
"Cn 297.0 394.000000 204.000000 35.000000\n",
43+
"D 0.0 0.176448 0.256805 0.365095\n",
44+
"Dn 309.0 370.000000 211.000000 40.000000\n",
45+
"E 0.0 0.152698 0.257873 0.376062\n",
46+
"En 273.0 319.000000 245.000000 93.000000\n",
47+
"F 0.0 0.179143 0.261386 0.354643\n",
48+
"Fn 200.0 237.000000 265.000000 228.000000\n"
49+
]
50+
}
51+
],
52+
"source": [
53+
"# 1> 数据预处理 \n",
54+
"\n",
55+
"# 1数据清洗\n",
56+
"# 2属性规约\n",
57+
"# 3数据变换\n",
58+
"# （1）属性构造\n",
59+
"# （2）数据离散化\n",
60+
"\n",
61+
"# -*- coding:utf-8 -*-\n",
62+
"from __future__ import print_function\n",
63+
"import pandas as pd\n",
64+
"from pandas import DataFrame,Series\n",
65+
"from sklearn.cluster import KMeans#导入K均值聚类算法\n",
66+
"\n",
67+
"datafile = 'data.xls'\n",
68+
"resultfile = 'data_processed.xlsx'\n",
69+
"\n",
70+
"typelabel = {u'肝气郁结证型系数':'A',u'热毒蕴结证型系数':'B',u'冲任失调证型系数':'C',u'气血两虚证型系数':'D',u'脾胃虚弱证型系数':'E',u'肝肾阴虚证型系数':'F'}\n",
71+
"\n",
72+
"k = 4 #需要进行的聚类类别数\n",
73+
"\n",
74+
"#读取文件进行聚类分析\n",
75+
"data = pd.read_excel(datafile)\n",
76+
"keys = list(typelabel.keys())\n",
77+
"result = DataFrame()\n",
78+
"\n",
79+
"for i in range(len(keys)):\n",
80+
" #调用k-means算法 进行聚类\n",
81+
" print(u'正在进行%s的聚类' % keys[i])\n",
82+
" kmodel = KMeans(n_clusters = k, n_jobs = 4) # n_job是线程数，根据自己电脑本身来调节\n",
83+
" kmodel.fit(data[[keys[i]]].as_matrix())# 训练模型\n",
84+
"# kmodel.fit(data[[keys[i]]]) # 不转成矩阵形式结果一样\n",
85+
"#KMeans(algorithm='auto', copy_x=True, init='k-means++', max_iter=300,\n",
86+
"# n_clusters=4, n_init=10, n_jobs=4, precompute_distances='auto',\n",
87+
"# random_state=None, tol=0.0001, verbose=0)\n",
88+
" \n",
89+
" r1 = DataFrame(kmodel.cluster_centers_, columns = [typelabel[keys[i]]]) # 聚类中心\n",
90+
" r2 = Series(kmodel.labels_).value_counts() #分类统计\n",
91+
" r2 = DataFrame(r2,columns = [typelabel[keys[i]]+'n'])# 转成DataFrame格式，记录各个类别的数目\n",
92+
" r = pd.concat([r1,r2], axis=1).sort_values(typelabel[keys[i]])\n",
93+
" r.index = range(1,5)\n",
94+
" r[typelabel[keys[i]]] = pd.rolling_mean(r[typelabel[keys[i]]],2) # rolling_mean用来计算相邻两列的均值，以此作为边界点\n",
95+
" r[typelabel[keys[i]]][1] = 0.0 # 将原来的聚类中心改成边界点\n",
96+
" result = result.append(r.T)\n",
97+
"result = result.sort_index() # 以index排序，以ABCDEF排序\n",
98+
"result.to_excel(resultfile)\n",
99+
" \n",
100+
"print (result)\n",
101+
"# '''\n",
102+
"# 1 2 3 4\n",
103+
"# A 0.0 0.178698 0.257724 0.351843\n",
104+
"# An 240.0 356.000000 281.000000 53.000000\n",
105+
"# B 0.0 0.150766 0.296631 0.489705\n",
106+
"# Bn 325.0 396.000000 180.000000 29.000000\n",
107+
"# C 0.0 0.202149 0.289061 0.423537\n",
108+
"# Cn 297.0 394.000000 204.000000 35.000000\n",
109+
"# D 0.0 0.172049 0.251583 0.359353\n",
110+
"# Dn 283.0 375.000000 228.000000 44.000000\n",
111+
"# E 0.0 0.152698 0.257762 0.375661\n",
112+
"# En 273.0 319.000000 244.000000 94.000000\n",
113+
"# F 0.0 0.179143 0.261386 0.354643\n",
114+
"# Fn 200.0 237.000000 265.000000 228.000000\n",
115+
"# '''\n"
116+
]
117+
},
118+
{
119+
"cell_type": "code",
120+
"execution_count": 2,
121+
"metadata": {},
122+
"outputs": [
123+
{
124+
"data": {
125+
"text/html": [
126+
"<div>\n",
127+
"<style>\n",
128+
" .dataframe thead tr:only-child th {\n",
129+
" text-align: right;\n",
130+
" }\n",
131+
"\n",
132+
" .dataframe thead th {\n",
133+
" text-align: left;\n",
134+
" }\n",
135+
"\n",
136+
" .dataframe tbody tr th {\n",
137+
" vertical-align: top;\n",
138+
" }\n",
139+
"</style>\n",
140+
"<table border=\"1\" class=\"dataframe\">\n",
141+
" <thead>\n",
142+
" <tr style=\"text-align: right;\">\n",
143+
" <th></th>\n",
144+
" <th>肝气郁结证型系数</th>\n",
145+
" <th>热毒蕴结证型系数</th>\n",
146+
" <th>冲任失调证型系数</th>\n",
147+
" <th>气血两虚证型系数</th>\n",
148+
" <th>脾胃虚弱证型系数</th>\n",
149+
" <th>肝肾阴虚证型系数</th>\n",
150+
" <th>病程阶段</th>\n",
151+
" <th>TNM分期</th>\n",
152+
" <th>转移部位</th>\n",
153+
" <th>确诊后几年发现转移</th>\n",
154+
" </tr>\n",
155+
" </thead>\n",
156+
" <tbody>\n",
157+
" <tr>\n",
158+
" <th>0</th>\n",
159+
" <td>0.056</td>\n",
160+
" <td>0.460</td>\n",
161+
" <td>0.281</td>\n",
162+
" <td>0.352</td>\n",
163+
" <td>0.119</td>\n",
164+
" <td>0.350</td>\n",
165+
" <td>S4</td>\n",
166+
" <td>H4</td>\n",
167+
" <td>R1</td>\n",
168+
" <td>J1</td>\n",
169+
" </tr>\n",
170+
" <tr>\n",
171+
" <th>1</th>\n",
172+
" <td>0.488</td>\n",
173+
" <td>0.099</td>\n",
174+
" <td>0.283</td>\n",
175+
" <td>0.333</td>\n",
176+
" <td>0.116</td>\n",
177+
" <td>0.293</td>\n",
178+
" <td>S4</td>\n",
179+
" <td>H4</td>\n",
180+
" <td>R1</td>\n",
181+
" <td>J1</td>\n",
182+
" </tr>\n",
183+
" <tr>\n",
184+
" <th>2</th>\n",
185+
" <td>0.107</td>\n",
186+
" <td>0.008</td>\n",
187+
" <td>0.204</td>\n",
188+
" <td>0.150</td>\n",
189+
" <td>0.032</td>\n",
190+
" <td>0.159</td>\n",
191+
" <td>S4</td>\n",
192+
" <td>H4</td>\n",
193+
" <td>R2</td>\n",
194+
" <td>J2</td>\n",
195+
" </tr>\n",
196+
" <tr>\n",
197+
" <th>3</th>\n",
198+
" <td>0.322</td>\n",
199+
" <td>0.208</td>\n",
200+
" <td>0.305</td>\n",
201+
" <td>0.130</td>\n",
202+
" <td>0.184</td>\n",
203+
" <td>0.317</td>\n",
204+
" <td>S4</td>\n",
205+
" <td>H4</td>\n",
206+
" <td>R2</td>\n",
207+
" <td>J1</td>\n",
208+
" </tr>\n",
209+
" <tr>\n",
210+
" <th>4</th>\n",
211+
" <td>0.242</td>\n",
212+
" <td>0.280</td>\n",
213+
" <td>0.131</td>\n",
214+
" <td>0.210</td>\n",
215+
" <td>0.191</td>\n",
216+
" <td>0.351</td>\n",
217+
" <td>S4</td>\n",
218+
" <td>H4</td>\n",
219+
" <td>R2R5</td>\n",
220+
" <td>J1</td>\n",
221+
" </tr>\n",
222+
" </tbody>\n",
223+
"</table>\n",
224+
"</div>"
225+
],
226+
"text/plain": [
227+
" 肝气郁结证型系数 热毒蕴结证型系数 冲任失调证型系数 气血两虚证型系数 脾胃虚弱证型系数 肝肾阴虚证型系数 病程阶段 TNM分期 \\\n",
228+
"0 0.056 0.460 0.281 0.352 0.119 0.350 S4 H4 \n",
229+
"1 0.488 0.099 0.283 0.333 0.116 0.293 S4 H4 \n",
230+
"2 0.107 0.008 0.204 0.150 0.032 0.159 S4 H4 \n",
231+
"3 0.322 0.208 0.305 0.130 0.184 0.317 S4 H4 \n",
232+
"4 0.242 0.280 0.131 0.210 0.191 0.351 S4 H4 \n",
233+
"\n",
234+
" 转移部位 确诊后几年发现转移 \n",
235+
"0 R1 J1 \n",
236+
"1 R1 J1 \n",
237+
"2 R2 J2 \n",
238+
"3 R2 J1 \n",
239+
"4 R2R5 J1 "
240+
]
241+
},
242+
"execution_count": 2,
243+
"metadata": {},
244+
"output_type": "execute_result"
245+
}
246+
],
247+
"source": [
248+
"# 2>划分原始数据中的类别\n",
249+
"import pandas as pd\n",
250+
"data.head()"
251+
]
252+
},
253+
{
254+
"cell_type": "code",
255+
"execution_count": 3,
256+
"metadata": {},
257+
"outputs": [
258+
{
259+
"data": {
260+
"text/html": [
261+
"<div>\n",
262+
"<style>\n",
263+
" .dataframe thead tr:only-child th {\n",
264+
" text-align: right;\n",
265+
" }\n",
266+
"\n",
267+
" .dataframe thead th {\n",
268+
" text-align: left;\n",
269+
" }\n",
270+
"\n",
271+
" .dataframe tbody tr th {\n",
272+
" vertical-align: top;\n",
273+
" }\n",
274+
"</style>\n",
275+
"<table border=\"1\" class=\"dataframe\">\n",
276+
" <thead>\n",
277+
" <tr style=\"text-align: right;\">\n",
278+
" <th></th>\n",
279+
" <th>肝气郁结证型系数</th>\n",
280+
" <th>热毒蕴结证型系数</th>\n",
281+
" <th>冲任失调证型系数</th>\n",
282+
" <th>气血两虚证型系数</th>\n",
283+
" <th>脾胃虚弱证型系数</th>\n",
284+
" <th>肝肾阴虚证型系数</th>\n",
285+
" </tr>\n",
286+
" </thead>\n",
287+
" <tbody>\n",
288+
" <tr>\n",
289+
" <th>0</th>\n",
290+
" <td>A1</td>\n",
291+
" <td>B4</td>\n",
292+
" <td>C2</td>\n",
293+
" <td>D3</td>\n",
294+
" <td>E1</td>\n",
295+
" <td>F3</td>\n",
296+
" </tr>\n",
297+
" <tr>\n",
298+
" <th>1</th>\n",
299+
" <td>A4</td>\n",
300+
" <td>B1</td>\n",
301+
" <td>C2</td>\n",
302+
" <td>D3</td>\n",
303+
" <td>E1</td>\n",
304+
" <td>F3</td>\n",
305+
" </tr>\n",
306+
" <tr>\n",
307+
" <th>2</th>\n",
308+
" <td>A1</td>\n",
309+
" <td>B1</td>\n",
310+
" <td>C2</td>\n",
311+
" <td>D1</td>\n",
312+
" <td>E1</td>\n",
313+
" <td>F1</td>\n",
314+
" </tr>\n",
315+
" <tr>\n",
316+
" <th>3</th>\n",
317+
" <td>A3</td>\n",
318+
" <td>B2</td>\n",
319+
" <td>C3</td>\n",
320+
" <td>D1</td>\n",
321+
" <td>E2</td>\n",
322+
" <td>F3</td>\n",
323+
" </tr>\n",
324+
" <tr>\n",
325+
" <th>4</th>\n",
326+
" <td>A2</td>\n",
327+
" <td>B2</td>\n",
328+
" <td>C1</td>\n",
329+
" <td>D2</td>\n",
330+
" <td>E2</td>\n",
331+
" <td>F3</td>\n",
332+
" </tr>\n",
333+
" </tbody>\n",
334+
"</table>\n",
335+
"</div>"
336+
],
337+
"text/plain": [
338+
" 肝气郁结证型系数 热毒蕴结证型系数 冲任失调证型系数 气血两虚证型系数 脾胃虚弱证型系数 肝肾阴虚证型系数\n",
339+
"0 A1 B4 C2 D3 E1 F3\n",
340+
"1 A4 B1 C2 D3 E1 F3\n",
341+
"2 A1 B1 C2 D1 E1 F1\n",
342+
"3 A3 B2 C3 D1 E2 F3\n",
343+
"4 A2 B2 C1 D2 E2 F3"
344+
]
345+
},
346+
"execution_count": 3,
347+
"metadata": {},
348+
"output_type": "execute_result"
349+
}
350+
],
351+
"source": [
352+
"# 将分类后数据进行处理（*****）\n",
353+
"data_cut = DataFrame(columns = data.columns[:6])\n",
354+
"types = ['A','B','C','D','E','F']\n",
355+
"num = ['1','2','3','4']\n",
356+
"for i in range(len(data_cut.columns)):\n",
357+
" value = list(data.iloc[:,i])\n",
358+
" bins = list(result[(2*i):(2*i+1)].values[0])\n",
359+
" bins.append(1)\n",
360+
" names = [str(x)+str(y) for x in types for y in num]\n",
361+
" group_names = names[4*i:4*(i+1)]\n",
362+
" cats = pd.cut(value,bins,labels=group_names,right=False)\n",
363+
" data_cut.iloc[:,i] = cats\n",
364+
"data_cut.to_excel('apriori.xlsx')\n",
365+
"data_cut.head()"
366+
]
367+
},
368+
{
369+
"cell_type": "code",
370+
"execution_count": null,
371+
"metadata": {
372+
"collapsed": true
373+
},
374+
"outputs": [],
375+
"source": []
376+
}
377+
],
378+
"metadata": {
379+
"kernelspec": {
380+
"display_name": "Python 2",
381+
"language": "python",
382+
"name": "python2"
383+
},
384+
"language_info": {
385+
"codemirror_mode": {
386+
"name": "ipython",
387+
"version": 2
388+
},
389+
"file_extension": ".py",
390+
"mimetype": "text/x-python",
391+
"name": "python",
392+
"nbconvert_exporter": "python",
393+
"pygments_lexer": "ipython2",
394+
"version": "2.7.13"
395+
}
396+
},
397+
"nbformat": 4,
398+
"nbformat_minor": 2
399+
}

0 commit comments

Comments
 (0)